2024-04-27T01:53:17.409 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-27T01:53:17.409 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --archive /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530 --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} --name teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530/orig.config.yaml 2024-04-27T01:53:17.455 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-27T01:53:17.543 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} email: ceph-qa@ceph.com first_in_suite: false job_id: '7675530' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a workunit: branch: squid sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 4854 sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a sleep_before_teardown: 0 subset: 40/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a targets: smithi026.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEOKYw2n/szPDKE7tDEvlUO83CM+TUCJziUZclbG41ILu4TsHqqGHvRjg81llNjlYlm2nBJp5cE2ExDlo3hmBo8= smithi151.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL49wT1Z7fqXO+J0pLl+liUsB0jvEFn0w9ec2PEMaNsTR2YnYH8Qo3tU94ihcS0t5TnvIA7y2uIT7595p8dLfJo= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_21:08:03 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-27T01:53:17.543 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa; will attempt to use it 2024-04-27T01:53:17.544 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks 2024-04-27T01:53:17.544 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-27T01:53:17.546 INFO:teuthology.task.internal:Checking packages... 2024-04-27T01:53:17.567 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a' 2024-04-27T01:53:17.567 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-27T01:53:17.567 INFO:teuthology.packaging:ref: None 2024-04-27T01:53:17.567 INFO:teuthology.packaging:tag: None 2024-04-27T01:53:17.568 INFO:teuthology.packaging:branch: squid 2024-04-27T01:53:17.568 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T01:53:17.568 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-27T01:53:17.787 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2447.gb22e2ebd 2024-04-27T01:53:17.789 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-27T01:53:17.795 INFO:teuthology.task.internal:no buildpackages task found 2024-04-27T01:53:17.796 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-27T01:53:17.864 INFO:teuthology.task.internal:Saving configuration 2024-04-27T01:53:17.880 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-27T01:53:17.895 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-27T01:53:17.922 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-27 01:48:00.132522', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEOKYw2n/szPDKE7tDEvlUO83CM+TUCJziUZclbG41ILu4TsHqqGHvRjg81llNjlYlm2nBJp5cE2ExDlo3hmBo8='} 2024-04-27T01:53:17.944 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-27 01:48:00.133866', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL49wT1Z7fqXO+J0pLl+liUsB0jvEFn0w9ec2PEMaNsTR2YnYH8Qo3tU94ihcS0t5TnvIA7y2uIT7595p8dLfJo='} 2024-04-27T01:53:17.944 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-27T01:53:17.966 INFO:teuthology.task.internal:roles: ubuntu@smithi026.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-27T01:53:17.966 INFO:teuthology.task.internal:roles: ubuntu@smithi151.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-27T01:53:17.966 INFO:teuthology.run_tasks:Running task console_log... 2024-04-27T01:53:18.101 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb17990c0d0>, signals=[15]) 2024-04-27T01:53:18.102 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-27T01:53:18.152 INFO:teuthology.task.internal:Opening connections... 2024-04-27T01:53:18.152 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi026.front.sepia.ceph.com 2024-04-27T01:53:18.154 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-27T01:53:18.228 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi151.front.sepia.ceph.com 2024-04-27T01:53:18.229 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-27T01:53:18.304 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-27T01:53:18.312 DEBUG:teuthology.orchestra.run.smithi026:> uname -m 2024-04-27T01:53:18.331 INFO:teuthology.orchestra.run.smithi026.stdout:x86_64 2024-04-27T01:53:18.331 DEBUG:teuthology.orchestra.run.smithi026:> cat /etc/os-release 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:NAME="CentOS Stream" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION="9" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:ID="centos" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:ID_LIKE="rhel fedora" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION_ID="9" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:PLATFORM_ID="platform:el9" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:ANSI_COLOR="0;31" 2024-04-27T01:53:18.389 INFO:teuthology.orchestra.run.smithi026.stdout:LOGO="fedora-logo-icon" 2024-04-27T01:53:18.390 INFO:teuthology.orchestra.run.smithi026.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-27T01:53:18.390 INFO:teuthology.orchestra.run.smithi026.stdout:HOME_URL="https://centos.org/" 2024-04-27T01:53:18.390 INFO:teuthology.orchestra.run.smithi026.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-27T01:53:18.390 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-27T01:53:18.390 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-27T01:53:18.390 INFO:teuthology.lock.ops:Updating smithi026.front.sepia.ceph.com on lock server 2024-04-27T01:53:18.417 DEBUG:teuthology.orchestra.run.smithi151:> uname -m 2024-04-27T01:53:18.435 INFO:teuthology.orchestra.run.smithi151.stdout:x86_64 2024-04-27T01:53:18.435 DEBUG:teuthology.orchestra.run.smithi151:> cat /etc/os-release 2024-04-27T01:53:18.492 INFO:teuthology.orchestra.run.smithi151.stdout:NAME="CentOS Stream" 2024-04-27T01:53:18.492 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION="9" 2024-04-27T01:53:18.492 INFO:teuthology.orchestra.run.smithi151.stdout:ID="centos" 2024-04-27T01:53:18.492 INFO:teuthology.orchestra.run.smithi151.stdout:ID_LIKE="rhel fedora" 2024-04-27T01:53:18.492 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION_ID="9" 2024-04-27T01:53:18.492 INFO:teuthology.orchestra.run.smithi151.stdout:PLATFORM_ID="platform:el9" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:ANSI_COLOR="0;31" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:LOGO="fedora-logo-icon" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:HOME_URL="https://centos.org/" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-27T01:53:18.493 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-27T01:53:18.493 INFO:teuthology.lock.ops:Updating smithi151.front.sepia.ceph.com on lock server 2024-04-27T01:53:18.516 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-27T01:53:18.524 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-27T01:53:18.552 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-27T01:53:18.552 DEBUG:teuthology.orchestra.run.smithi026:> test '!' -e /home/ubuntu/cephtest 2024-04-27T01:53:18.555 DEBUG:teuthology.orchestra.run.smithi151:> test '!' -e /home/ubuntu/cephtest 2024-04-27T01:53:18.571 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-27T01:53:18.743 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-27T01:53:18.743 DEBUG:teuthology.orchestra.run.smithi026:> test -z $(ls -A /var/lib/ceph) 2024-04-27T01:53:18.746 DEBUG:teuthology.orchestra.run.smithi151:> test -z $(ls -A /var/lib/ceph) 2024-04-27T01:53:18.779 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-27T01:53:18.867 INFO:teuthology.run_tasks:Running task kernel... 2024-04-27T01:53:18.883 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-27T01:53:18.883 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-27T01:53:18.884 DEBUG:teuthology.orchestra.run.smithi026:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-27T01:53:18.884 DEBUG:teuthology.orchestra.run.smithi151:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-27T01:53:18.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T01:53:18.901 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2024-04-27T01:53:18.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T01:53:18.902 DEBUG:teuthology.orchestra.run.smithi026:> uname -r 2024-04-27T01:53:18.958 INFO:teuthology.orchestra.run.smithi151.stdout:5.14.0-437.el9.x86_64 2024-04-27T01:53:18.958 INFO:teuthology.task.kernel:Running kernel on smithi151: 5.14.0-437.el9.x86_64 2024-04-27T01:53:18.958 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum install -y kernel 2024-04-27T01:53:18.962 INFO:teuthology.orchestra.run.smithi026.stdout:5.14.0-437.el9.x86_64 2024-04-27T01:53:18.962 INFO:teuthology.task.kernel:Running kernel on smithi026: 5.14.0-437.el9.x86_64 2024-04-27T01:53:18.963 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum install -y kernel 2024-04-27T01:53:21.207 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.0 MB 00:00 2024-04-27T01:53:21.419 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - BaseOS 9.9 MB/s | 8.0 MB 00:00 2024-04-27T01:53:24.047 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - AppStream 12 MB/s | 19 MB 00:01 2024-04-27T01:53:24.941 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - AppStream 7.1 MB/s | 19 MB 00:02 2024-04-27T01:53:28.253 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.2 MB 00:00 2024-04-27T01:53:29.183 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - CRB 8.9 MB/s | 6.2 MB 00:00 2024-04-27T01:53:29.662 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - Extras packages 77 kB/s | 16 kB 00:00 2024-04-27T01:53:31.663 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-04-27T01:53:31.717 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - Extras packages 14 kB/s | 16 kB 00:01 2024-04-27T01:53:32.835 INFO:teuthology.orchestra.run.smithi151.stdout:Extra Packages for Enterprise Linux 21 MB/s | 21 MB 00:00 2024-04-27T01:53:36.930 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-04-27T01:53:37.927 INFO:teuthology.orchestra.run.smithi151.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-27T01:53:38.887 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-27T01:53:38.887 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-27T01:53:38.976 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-04-27T01:53:38.977 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2024-04-27T01:53:38.977 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-04-27T01:53:39.078 DEBUG:teuthology.orchestra.run.smithi026:> echo no | sudo yum reinstall kernel || true 2024-04-27T01:53:39.600 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:03 ago on Sat 27 Apr 2024 01:53:36 AM UTC. 2024-04-27T01:53:39.731 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-04-27T01:53:39.741 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-27T01:53:39.741 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2024-04-27T01:53:39.741 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout:Reinstalling: 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 255 k 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 0 2024-04-27T01:53:39.742 INFO:teuthology.orchestra.run.smithi026.stderr:Operation aborted. 2024-04-27T01:53:39.793 INFO:teuthology.orchestra.run.smithi026.stdout:Is this ok [y/N]: 2024-04-27T01:53:39.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum reinstall -y kernel || true 2024-04-27T01:53:39.824 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-27T01:53:39.824 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-27T01:53:39.917 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-27T01:53:39.919 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-27T01:53:39.919 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-27T01:53:40.032 DEBUG:teuthology.orchestra.run.smithi151:> echo no | sudo yum reinstall kernel || true 2024-04-27T01:53:40.312 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:04 ago on Sat 27 Apr 2024 01:53:36 AM UTC. 2024-04-27T01:53:40.433 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-04-27T01:53:40.434 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-27T01:53:40.434 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2024-04-27T01:53:40.434 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-27T01:53:40.434 INFO:teuthology.orchestra.run.smithi026.stdout:Reinstalling: 2024-04-27T01:53:40.434 INFO:teuthology.orchestra.run.smithi026.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-27T01:53:40.434 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T01:53:40.435 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-04-27T01:53:40.435 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-27T01:53:40.435 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T01:53:40.435 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 255 k 2024-04-27T01:53:40.435 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 0 2024-04-27T01:53:40.435 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2024-04-27T01:53:40.534 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:03 ago on Sat 27 Apr 2024 01:53:37 AM UTC. 2024-04-27T01:53:40.659 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-27T01:53:40.667 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-27T01:53:40.667 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2024-04-27T01:53:40.667 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-27T01:53:40.667 INFO:teuthology.orchestra.run.smithi151.stdout:Reinstalling: 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 255 k 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 0 2024-04-27T01:53:40.668 INFO:teuthology.orchestra.run.smithi151.stderr:Operation aborted. 2024-04-27T01:53:40.715 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.1 MB/s | 255 kB 00:00 2024-04-27T01:53:40.715 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2024-04-27T01:53:40.715 INFO:teuthology.orchestra.run.smithi026.stdout:Total 908 kB/s | 255 kB 00:00 2024-04-27T01:53:40.722 INFO:teuthology.orchestra.run.smithi151.stdout:Is this ok [y/N]: 2024-04-27T01:53:40.723 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum reinstall -y kernel || true 2024-04-27T01:53:40.734 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2024-04-27T01:53:40.925 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2024-04-27T01:53:40.925 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2024-04-27T01:53:40.931 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2024-04-27T01:53:40.931 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2024-04-27T01:53:41.178 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2024-04-27T01:53:41.223 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:04 ago on Sat 27 Apr 2024 01:53:37 AM UTC. 2024-04-27T01:53:41.242 INFO:teuthology.orchestra.run.smithi026.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-27T01:53:41.242 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-27T01:53:41.339 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-27T01:53:41.340 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-27T01:53:41.340 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2024-04-27T01:53:41.340 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-27T01:53:41.340 INFO:teuthology.orchestra.run.smithi151.stdout:Reinstalling: 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 255 k 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 0 2024-04-27T01:53:41.341 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-27T01:53:41.620 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.2 MB/s | 255 kB 00:00 2024-04-27T01:53:41.621 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-27T01:53:41.621 INFO:teuthology.orchestra.run.smithi151.stdout:Total 909 kB/s | 255 kB 00:00 2024-04-27T01:53:41.637 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-27T01:53:41.808 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-27T01:53:41.808 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-27T01:53:41.813 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-27T01:53:41.813 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-27T01:53:41.907 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-27T01:53:41.908 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-27T01:53:41.935 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-27T01:53:42.022 INFO:teuthology.orchestra.run.smithi151.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-27T01:53:42.022 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-27T01:53:42.550 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-27T01:53:42.551 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T01:53:42.551 INFO:teuthology.orchestra.run.smithi026.stdout:Reinstalled: 2024-04-27T01:53:42.551 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-27T01:53:42.551 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T01:53:42.551 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-04-27T01:53:42.729 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1 2024-04-27T01:53:42.775 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-27T01:53:42.776 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-27T01:53:42.776 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-27T01:53:42.776 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-27T01:53:42.777 DEBUG:teuthology.orchestra.run.smithi026:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-27T01:53:42.812 INFO:teuthology.orchestra.run.smithi026.stdout:ttyS1 2024-04-27T01:53:42.820 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-27T01:53:42.821 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-27T01:53:42.837 DEBUG:teuthology.parallel:result is None 2024-04-27T01:53:43.483 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-27T01:53:43.483 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:53:43.484 INFO:teuthology.orchestra.run.smithi151.stdout:Reinstalled: 2024-04-27T01:53:43.484 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-27T01:53:43.484 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:53:43.484 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-27T01:53:43.674 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2024-04-27T01:53:43.721 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-27T01:53:43.721 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-27T01:53:43.721 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-27T01:53:43.721 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-27T01:53:43.721 DEBUG:teuthology.orchestra.run.smithi151:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-27T01:53:43.755 INFO:teuthology.orchestra.run.smithi151.stdout:ttyS1 2024-04-27T01:53:43.779 DEBUG:teuthology.parallel:result is None 2024-04-27T01:53:43.779 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-27T01:53:43.787 INFO:teuthology.task.internal:Creating test directory... 2024-04-27T01:53:43.787 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-27T01:53:43.789 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-27T01:53:43.834 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-27T01:53:43.851 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-27T01:53:43.971 INFO:teuthology.task.internal:Creating archive directory... 2024-04-27T01:53:43.971 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-27T01:53:43.974 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-27T01:53:44.010 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-27T01:53:44.030 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-27T01:53:44.031 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-27T01:53:44.048 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-27T01:53:44.081 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-27T01:53:44.085 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-27T01:53:44.090 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-27T01:53:44.096 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-27T01:53:44.098 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-27T01:53:44.105 INFO:teuthology.task.internal:Configuring sudo... 2024-04-27T01:53:44.105 DEBUG:teuthology.orchestra.run.smithi026:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-27T01:53:44.140 DEBUG:teuthology.orchestra.run.smithi151:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-27T01:53:44.166 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-27T01:53:44.249 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-27T01:53:44.249 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-27T01:53:44.251 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-27T01:53:44.267 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-27T01:53:44.351 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-27T01:53:44.405 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-27T01:53:44.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-27T01:53:44.531 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:53:44.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-27T01:53:44.597 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-27T01:53:44.634 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-27T01:53:44.681 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-27T01:53:44.753 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-27T01:53:44.801 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-27T01:53:44.801 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-27T01:53:44.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo service rsyslog restart 2024-04-27T01:53:44.867 DEBUG:teuthology.orchestra.run.smithi151:> sudo service rsyslog restart 2024-04-27T01:53:44.902 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-27T01:53:44.953 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-27T01:53:45.397 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-27T01:53:45.404 INFO:teuthology.task.internal:Starting timer... 2024-04-27T01:53:45.404 INFO:teuthology.run_tasks:Running task pcp... 2024-04-27T01:53:45.642 INFO:teuthology.run_tasks:Running task selinux... 2024-04-27T01:53:45.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo service auditd rotate 2024-04-27T01:53:45.728 INFO:teuthology.orchestra.run.smithi026.stdout:Rotating logs: 2024-04-27T01:53:45.730 DEBUG:teuthology.orchestra.run.smithi151:> sudo service auditd rotate 2024-04-27T01:53:45.777 INFO:teuthology.orchestra.run.smithi151.stdout:Rotating logs: 2024-04-27T01:53:45.778 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-27T01:53:45.778 DEBUG:teuthology.orchestra.run.smithi026:> /usr/sbin/getenforce 2024-04-27T01:53:45.799 INFO:teuthology.orchestra.run.smithi026.stdout:Permissive 2024-04-27T01:53:45.799 DEBUG:teuthology.orchestra.run.smithi151:> /usr/sbin/getenforce 2024-04-27T01:53:45.816 INFO:teuthology.orchestra.run.smithi151.stdout:Permissive 2024-04-27T01:53:45.816 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi026.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi151.front.sepia.ceph.com': 'permissive'} 2024-04-27T01:53:45.816 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-27T01:53:45.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T01:53:45.864 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-27T01:53:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T01:53:45.888 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-27T01:53:45.889 DEBUG:teuthology.orchestra.run.smithi026:> sudo /usr/sbin/setenforce permissive 2024-04-27T01:53:45.929 DEBUG:teuthology.orchestra.run.smithi151:> sudo /usr/sbin/setenforce permissive 2024-04-27T01:53:45.960 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-27T01:53:45.969 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-27T01:53:45.974 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-27T01:53:46.098 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-27T01:53:46.112 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-27T01:53:46.113 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi026.front.sepia.ceph.com,smithi151.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-27T01:57:17.807 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi026.front.sepia.ceph.com'), Remote(name='ubuntu@smithi151.front.sepia.ceph.com')] 2024-04-27T01:57:17.808 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-04-27T01:57:17.809 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-27T01:57:17.887 DEBUG:teuthology.orchestra.run.smithi026:> true 2024-04-27T01:57:17.966 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-04-27T01:57:17.966 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-27T01:57:17.967 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-27T01:57:18.040 DEBUG:teuthology.orchestra.run.smithi151:> true 2024-04-27T01:57:18.075 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-27T01:57:18.076 INFO:teuthology.run_tasks:Running task clock... 2024-04-27T01:57:18.086 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-27T01:57:18.086 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-27T01:57:18.086 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-27T01:57:18.089 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-27T01:57:18.089 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-27T01:57:18.114 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-27T01:57:18.128 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-27T01:57:18.142 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-27T01:57:18.152 INFO:teuthology.orchestra.run.smithi026.stderr:sudo: ntpd: command not found 2024-04-27T01:57:18.155 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-27T01:57:18.163 INFO:teuthology.orchestra.run.smithi026.stdout:506 Cannot talk to daemon 2024-04-27T01:57:18.177 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-27T01:57:18.177 INFO:teuthology.orchestra.run.smithi151.stderr:sudo: ntpd: command not found 2024-04-27T01:57:18.187 INFO:teuthology.orchestra.run.smithi151.stdout:506 Cannot talk to daemon 2024-04-27T01:57:18.194 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-27T01:57:18.200 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-27T01:57:18.213 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-27T01:57:18.248 INFO:teuthology.orchestra.run.smithi026.stderr:bash: line 1: ntpq: command not found 2024-04-27T01:57:18.251 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-27T01:57:18.251 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2024-04-27T01:57:18.251 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.251 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.251 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.251 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.272 INFO:teuthology.orchestra.run.smithi151.stderr:bash: line 1: ntpq: command not found 2024-04-27T01:57:18.276 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-27T01:57:18.276 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2024-04-27T01:57:18.276 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.276 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.276 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.276 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T01:57:18.277 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-27T01:57:18.376 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-27T01:57:18.376 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-27T01:57:18.377 INFO:tasks.cephadm:Cluster fsid is 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:18.377 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-27T01:57:18.377 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.26', 'mon.c': '[v2:172.21.15.26:3301,v1:172.21.15.26:6790]', 'mon.b': '172.21.15.151'} 2024-04-27T01:57:18.377 INFO:tasks.cephadm:First mon is mon.a on smithi026 2024-04-27T01:57:18.377 INFO:tasks.cephadm:First mgr is y 2024-04-27T01:57:18.377 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-27T01:57:18.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo hostname $(hostname -s) 2024-04-27T01:57:18.403 DEBUG:teuthology.orchestra.run.smithi151:> sudo hostname $(hostname -s) 2024-04-27T01:57:18.427 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-27T01:57:18.428 DEBUG:teuthology.orchestra.run.smithi026:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-27T01:57:18.659 INFO:teuthology.orchestra.run.smithi026.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 27 01:57 /home/ubuntu/cephtest/cephadm 2024-04-27T01:57:18.660 DEBUG:teuthology.orchestra.run.smithi151:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-27T01:57:18.825 INFO:teuthology.orchestra.run.smithi151.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 27 01:57 /home/ubuntu/cephtest/cephadm 2024-04-27T01:57:18.825 DEBUG:teuthology.orchestra.run.smithi026:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-27T01:57:18.842 DEBUG:teuthology.orchestra.run.smithi151:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-27T01:57:18.868 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-27T01:57:18.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-27T01:57:18.885 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-27T01:57:20.348 INFO:teuthology.orchestra.run.smithi151.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-27T01:57:20.556 INFO:teuthology.orchestra.run.smithi026.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-27T01:57:37.643 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2024-04-27T01:57:37.643 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-27T01:57:37.643 INFO:teuthology.orchestra.run.smithi151.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-27T01:57:37.643 INFO:teuthology.orchestra.run.smithi151.stdout: "repo_digests": [ 2024-04-27T01:57:37.643 INFO:teuthology.orchestra.run.smithi151.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-27T01:57:37.643 INFO:teuthology.orchestra.run.smithi151.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-27T01:57:37.643 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2024-04-27T01:57:37.644 INFO:teuthology.orchestra.run.smithi151.stdout:} 2024-04-27T01:57:38.002 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T01:57:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-27T01:57:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-27T01:57:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "repo_digests": [ 2024-04-27T01:57:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-27T01:57:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-27T01:57:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2024-04-27T01:57:38.003 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T01:57:38.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo mkdir -p /etc/ceph 2024-04-27T01:57:38.074 DEBUG:teuthology.orchestra.run.smithi151:> sudo mkdir -p /etc/ceph 2024-04-27T01:57:38.102 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 777 /etc/ceph 2024-04-27T01:57:38.126 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 777 /etc/ceph 2024-04-27T01:57:38.166 INFO:tasks.cephadm:Writing seed config... 2024-04-27T01:57:38.167 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-27T01:57:38.167 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-27T01:57:38.167 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-27T01:57:38.167 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-27T01:57:38.167 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-27T01:57:38.167 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-27T01:57:38.168 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-27T01:57:38.168 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-27T01:57:38.168 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-27T01:57:38.168 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:57:38.168 DEBUG:teuthology.orchestra.run.smithi026:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-27T01:57:38.183 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 7a5462be-0439-11ef-bc93-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-27T01:57:38.183 DEBUG:teuthology.orchestra.run.smithi026:mon.a> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service 2024-04-27T01:57:38.226 DEBUG:teuthology.orchestra.run.smithi026:mgr.y> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service 2024-04-27T01:57:38.269 INFO:tasks.cephadm:Bootstrapping... 2024-04-27T01:57:38.270 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 7a5462be-0439-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.26 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-27T01:57:38.468 INFO:teuthology.orchestra.run.smithi026.stderr:-------------------------------------------------------------------------------- 2024-04-27T01:57:38.469 INFO:teuthology.orchestra.run.smithi026.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '7a5462be-0439-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.26', '--skip-admin-label'] 2024-04-27T01:57:38.493 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: 4.9.4-dev 2024-04-27T01:57:38.495 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying podman|docker is present... 2024-04-27T01:57:38.516 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: 4.9.4-dev 2024-04-27T01:57:38.518 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying lvm2 is present... 2024-04-27T01:57:38.518 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying time synchronization is in place... 2024-04-27T01:57:38.525 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-27T01:57:38.531 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-27T01:57:38.538 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: enabled 2024-04-27T01:57:38.544 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: active 2024-04-27T01:57:38.545 INFO:teuthology.orchestra.run.smithi026.stderr:Unit chronyd.service is enabled and running 2024-04-27T01:57:38.545 INFO:teuthology.orchestra.run.smithi026.stderr:Repeating the final host check... 2024-04-27T01:57:38.566 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: 4.9.4-dev 2024-04-27T01:57:38.568 INFO:teuthology.orchestra.run.smithi026.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-27T01:57:38.568 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl is present 2024-04-27T01:57:38.568 INFO:teuthology.orchestra.run.smithi026.stderr:lvcreate is present 2024-04-27T01:57:38.574 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-27T01:57:38.580 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-27T01:57:38.587 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: enabled 2024-04-27T01:57:38.593 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: active 2024-04-27T01:57:38.594 INFO:teuthology.orchestra.run.smithi026.stderr:Unit chronyd.service is enabled and running 2024-04-27T01:57:38.594 INFO:teuthology.orchestra.run.smithi026.stderr:Host looks OK 2024-04-27T01:57:38.594 INFO:teuthology.orchestra.run.smithi026.stderr:Cluster fsid: 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:38.594 INFO:teuthology.orchestra.run.smithi026.stderr:Acquiring lock 140564700155760 on /run/cephadm/7a5462be-0439-11ef-bc93-c7b262605968.lock 2024-04-27T01:57:38.594 INFO:teuthology.orchestra.run.smithi026.stderr:Lock 140564700155760 acquired on /run/cephadm/7a5462be-0439-11ef-bc93-c7b262605968.lock 2024-04-27T01:57:38.595 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying IP 172.21.15.26 port 3300 ... 2024-04-27T01:57:38.595 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying IP 172.21.15.26 port 6789 ... 2024-04-27T01:57:38.595 INFO:teuthology.orchestra.run.smithi026.stderr:Base mon IP is 172.21.15.26, final addrv is [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2024-04-27T01:57:38.597 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.26 metric 100 2024-04-27T01:57:38.598 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.26 metric 100 2024-04-27T01:57:38.600 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-27T01:57:38.600 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-27T01:57:38.600 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-27T01:57:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-27T01:57:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-27T01:57:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-27T01:57:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-27T01:57:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fea5:cac0/64 scope link noprefixroute 2024-04-27T01:57:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-27T01:57:38.604 INFO:teuthology.orchestra.run.smithi026.stderr:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2024-04-27T01:57:38.604 INFO:teuthology.orchestra.run.smithi026.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-27T01:57:38.604 INFO:teuthology.orchestra.run.smithi026.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-27T01:57:38.711 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-27T01:57:39.114 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Getting image source signatures 2024-04-27T01:57:39.120 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-27T01:57:39.120 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-27T01:57:39.120 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-27T01:57:39.120 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-27T01:57:39.120 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-27T01:57:39.120 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-27T01:57:39.326 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Writing manifest to image destination 2024-04-27T01:57:39.476 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-27T01:57:39.826 INFO:teuthology.orchestra.run.smithi026.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-27T01:57:40.342 INFO:teuthology.orchestra.run.smithi026.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-27T01:57:40.342 INFO:teuthology.orchestra.run.smithi026.stderr:Extracting ceph user uid/gid from container image... 2024-04-27T01:57:40.749 INFO:teuthology.orchestra.run.smithi026.stderr:stat: 167 167 2024-04-27T01:57:41.196 INFO:teuthology.orchestra.run.smithi026.stderr:Creating initial keys... 2024-04-27T01:57:41.562 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph-authtool: AQCVWyxm4d2pHhAAcYSZGO4cpy7+BkzlkLeiMA== 2024-04-27T01:57:42.518 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph-authtool: AQCWWyxmdykLGRAA7YJ+9l3XMq+qpTMOpbVR6g== 2024-04-27T01:57:43.615 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph-authtool: AQCXWyxmVOXdIBAA1ozNwxmFQjheXetc5l0Cew== 2024-04-27T01:57:44.163 INFO:teuthology.orchestra.run.smithi026.stderr:Creating initial monmap... 2024-04-27T01:57:44.512 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-27T01:57:44.513 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-27T01:57:44.513 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:44.513 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-27T01:57:45.011 INFO:teuthology.orchestra.run.smithi026.stderr:monmaptool for a [v2:172.21.15.26:3300,v1:172.21.15.26:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-27T01:57:45.011 INFO:teuthology.orchestra.run.smithi026.stderr:setting min_mon_release = octopus 2024-04-27T01:57:45.011 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: set fsid to 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:45.011 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-27T01:57:45.011 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:57:45.011 INFO:teuthology.orchestra.run.smithi026.stderr:Creating mon... 2024-04-27T01:57:45.924 INFO:teuthology.orchestra.run.smithi026.stderr:create mon.a on 2024-04-27T01:57:46.133 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-27T01:57:46.348 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-7a5462be-0439-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-7a5462be-0439-11ef-bc93-c7b262605968.target. 2024-04-27T01:57:46.348 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-7a5462be-0439-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-7a5462be-0439-11ef-bc93-c7b262605968.target. 2024-04-27T01:57:46.653 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to reset failed state of unit ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-27T01:57:46.660 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/ceph-7a5462be-0439-11ef-bc93-c7b262605968.target.wants/ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-7a5462be-0439-11ef-bc93-c7b262605968@.service. 2024-04-27T01:57:47.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 podman[21491]: 2024-04-27 01:57:46.990530301 +0000 UTC m=+0.019063928 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-27T01:57:47.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 podman[21491]: 2024-04-27 01:57:47.109888836 +0000 UTC m=+0.138422448 container create a27d6fd97ea8465488344d896bc4a52905f2765d5269a42df7a0b88c51ecd603 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_CLEAN=True, version=8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-27T01:57:47.338 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-27T01:57:47.345 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-27T01:57:47.346 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-27T01:57:47.346 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to enable service . firewalld.service is not available 2024-04-27T01:57:47.346 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mon to start... 2024-04-27T01:57:47.346 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mon... 2024-04-27T01:57:47.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 podman[21491]: 2024-04-27 01:57:47.271073943 +0000 UTC m=+0.299607568 container init a27d6fd97ea8465488344d896bc4a52905f2765d5269a42df7a0b88c51ecd603 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, com.redhat.component=centos-stream-container, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.buildah.version=1.19.8, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, RELEASE=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, ceph=True) 2024-04-27T01:57:47.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 podman[21491]: 2024-04-27 01:57:47.274453976 +0000 UTC m=+0.302987591 container start a27d6fd97ea8465488344d896bc4a52905f2765d5269a42df7a0b88c51ecd603 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, architecture=x86_64, release=754, vcs-type=git, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container) 2024-04-27T01:57:47.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 ceph-mon[21510]: mkfs 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:47.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 ceph-mon[21510]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-27T01:57:47.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 bash[21491]: a27d6fd97ea8465488344d896bc4a52905f2765d5269a42df7a0b88c51ecd603 2024-04-27T01:57:47.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:47 smithi026 systemd[1]: Started Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: cluster: 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: id: 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: services: 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.57443s) 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: data: 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-27T01:57:47.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-27T01:57:47.868 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-27T01:57:47.868 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: pgs: 2024-04-27T01:57:47.868 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:48.410 INFO:teuthology.orchestra.run.smithi026.stderr:mon is available 2024-04-27T01:57:48.411 INFO:teuthology.orchestra.run.smithi026.stderr:Assimilating anything we can from ceph.conf... 2024-04-27T01:57:48.524 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:48 smithi026 ceph-mon[21510]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-27T01:57:48.524 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:48 smithi026 ceph-mon[21510]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} 2024-04-27T01:57:48.524 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:48 smithi026 ceph-mon[21510]: fsmap 2024-04-27T01:57:48.524 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:48 smithi026 ceph-mon[21510]: osdmap e1: 0 total, 0 up, 0 in 2024-04-27T01:57:48.525 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:48 smithi026 ceph-mon[21510]: mgrmap e1: no daemons active 2024-04-27T01:57:48.525 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:48 smithi026 ceph-mon[21510]: from='client.? 172.21.15.26:0/2193007901' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T01:57:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [global] 2024-04-27T01:57:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: fsid = 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2024-04-27T01:57:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-27T01:57:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-27T01:57:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [mgr] 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [osd] 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-27T01:57:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-27T01:57:49.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:49 smithi026 ceph-mon[21510]: from='client.? 172.21.15.26:0/1491264149' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-27T01:57:49.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:49 smithi026 ceph-mon[21510]: from='client.? 172.21.15.26:0/1491264149' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-27T01:57:49.449 INFO:teuthology.orchestra.run.smithi026.stderr:Generating new minimal ceph.conf... 2024-04-27T01:57:50.447 INFO:teuthology.orchestra.run.smithi026.stderr:Restarting the monitor... 2024-04-27T01:57:50.496 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:50 smithi026 ceph-mon[21510]: from='client.? 172.21.15.26:0/865425644' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:57:50.496 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:50 smithi026 systemd[1]: Stopping Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T01:57:50.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:50 smithi026 bash[21741]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-27T01:57:50.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:50 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a[21505]: 2024-04-27T01:57:50.627+0000 7f48b845a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T01:57:50.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:50 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a[21505]: 2024-04-27T01:57:50.627+0000 7f48b845a700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-27T01:57:51.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:50 smithi026 podman[21754]: 2024-04-27 01:57:50.833074533 +0000 UTC m=+0.308488830 container died a27d6fd97ea8465488344d896bc4a52905f2765d5269a42df7a0b88c51ecd603 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0) 2024-04-27T01:57:51.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 podman[21754]: 2024-04-27 01:57:51.199196873 +0000 UTC m=+0.674611159 container cleanup a27d6fd97ea8465488344d896bc4a52905f2765d5269a42df7a0b88c51ecd603 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, com.redhat.component=centos-stream-container, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, vcs-type=git, name=centos-stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.k8s.display-name=CentOS Stream 8, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, ceph=True, vendor=Red Hat, Inc., GIT_CLEAN=True) 2024-04-27T01:57:51.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 bash[21754]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a 2024-04-27T01:57:51.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 bash[21781]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-27T01:57:51.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 podman[21768]: 2024-04-27 01:57:51.467019592 +0000 UTC m=+0.628901129 container remove a27d6fd97ea8465488344d896bc4a52905f2765d5269a42df7a0b88c51ecd603 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-type=git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, distribution-scope=public) 2024-04-27T01:57:51.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-27T01:57:51.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 systemd[1]: Stopped Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T01:57:51.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 systemd[1]: Starting Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T01:57:51.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 podman[21848]: 2024-04-27 01:57:51.614954694 +0000 UTC m=+0.019017401 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-27T01:57:51.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 podman[21848]: 2024-04-27 01:57:51.743009606 +0000 UTC m=+0.147072284 container create f821e7b12391c592a84a0d3aa3af4a9dbf5233ad2ab0b0b4ac7ad15468949875 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, release=754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-27T01:57:51.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 podman[21848]: 2024-04-27 01:57:51.945524117 +0000 UTC m=+0.349586799 container init f821e7b12391c592a84a0d3aa3af4a9dbf5233ad2ab0b0b4ac7ad15468949875 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, vendor=Red Hat, Inc., version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream) 2024-04-27T01:57:51.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 podman[21848]: 2024-04-27 01:57:51.948797773 +0000 UTC m=+0.352860452 container start f821e7b12391c592a84a0d3aa3af4a9dbf5233ad2ab0b0b4ac7ad15468949875 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, version=8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream) 2024-04-27T01:57:51.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: set uid:gid to 167:167 (ceph:ceph) 2024-04-27T01:57:51.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-27T01:57:51.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: pidfile_write: ignore empty --pid-file 2024-04-27T01:57:51.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: load: jerasure load: lrc 2024-04-27T01:57:51.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: RocksDB version: 6.15.5 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Compile date Apr 18 2022 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: DB SUMMARY 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: DB Session ID: VG81I8X7HLH64VYCSNMM 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: CURRENT file: CURRENT 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: IDENTITY file: IDENTITY 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82631 ; 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.error_if_exists: 0 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.create_if_missing: 0 2024-04-27T01:57:51.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.paranoid_checks: 1 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.env: 0x558d740c6860 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.fs: Posix File System 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.info_log: 0x558d7633bdc0 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_file_opening_threads: 16 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.statistics: (nil) 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.use_fsync: 0 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_log_file_size: 0 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-27T01:57:51.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.keep_log_file_num: 1000 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.recycle_log_file_num: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.allow_fallocate: 1 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.allow_mmap_reads: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.allow_mmap_writes: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.use_direct_reads: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.create_missing_column_families: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.db_log_dir: 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-27T01:57:51.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-27T01:57:51.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-27T01:57:51.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.advise_random_on_open: 1 2024-04-27T01:57:51.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.db_write_buffer_size: 0 2024-04-27T01:57:51.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.write_buffer_manager: 0x558d7642c2a0 2024-04-27T01:57:51.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-27T01:57:51.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.rate_limiter: (nil) 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.wal_recovery_mode: 2 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.enable_thread_tracking: 0 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.enable_pipelined_write: 0 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.unordered_write: 0 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.row_cache: None 2024-04-27T01:57:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.wal_filter: None 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.allow_ingest_behind: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.preserve_deletes: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.two_write_queues: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.manual_wal_flush: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.atomic_flush: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.log_readahead_size: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.best_efforts_recovery: 0 2024-04-27T01:57:51.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.allow_data_in_errors: 0 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.db_host_id: __hostname__ 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_background_jobs: 2 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_background_compactions: -1 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_subcompactions: 1 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_total_wal_size: 0 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-27T01:57:51.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_open_files: -1 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bytes_per_sync: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_readahead_size: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_background_flushes: -1 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Compression algorithms supported: 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kZSTD supported: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kXpressCompression supported: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kLZ4HCCompression supported: 1 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kLZ4Compression supported: 1 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kBZip2Compression supported: 0 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kZlibCompression supported: 1 2024-04-27T01:57:52.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: kSnappyCompression supported: 1 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.merge_operator: 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_filter: None 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_filter_factory: None 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.sst_partitioner_factory: None 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558d76307d00) 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-27T01:57:52.001 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: hash_index_allow_collision: 1 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 1 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x558d76372170 2024-04-27T01:57:52.002 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-04-27T01:57:52.002 INFO:teuthology.orchestra.run.smithi026.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2024-04-27T01:57:52.004 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 4 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.write_buffer_size: 33554432 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_write_buffer_number: 2 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression: NoCompression 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression: Disabled 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.prefix_extractor: nullptr 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-27T01:57:52.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.num_levels: 7 2024-04-27T01:57:52.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-27T01:57:52.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-27T01:57:52.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-27T01:57:52.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression_opts.level: 32767 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression_opts.strategy: 0 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-27T01:57:52.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compression_opts.enabled: false 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.target_file_size_base: 67108864 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-27T01:57:52.008 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.arena_block_size: 4194304 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.disable_auto_compactions: 0 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-27T01:57:52.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.table_properties_collectors: 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.inplace_update_support: 0 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.bloom_locality: 0 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.max_successive_merges: 0 2024-04-27T01:57:52.010 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.paranoid_file_checks: 0 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.force_consistency_checks: 1 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.report_bg_io_stats: 0 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.ttl: 2592000 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.enable_blob_files: false 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.min_blob_size: 0 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.blob_file_size: 268435456 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-27T01:57:52.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714183071974741, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714183071975504, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79181, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77444, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71753, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714183071, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "41458d63-454e-4133-a9cc-5993f3632732", "db_session_id": "VG81I8X7HLH64VYCSNMM"}} 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714183071975833, "job": 1, "event": "recovery_finished"} 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x558d76358a80 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: DB pointer 0x558d76368000 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2024-04-27T01:57:52.012 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.4 0.00 0.00 1 0.000 0 0 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.4 0.00 0.00 1 0.000 0 0 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.4 0.00 0.00 1 0.000 0 0 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T01:57:52.013 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 154.4 0.00 0.00 1 0.000 0 0 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 24.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 24.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T01:57:52.014 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.4 0.00 0.00 1 0.000 0 0 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.4 0.00 0.00 1 0.000 0 0 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 154.4 0.00 0.00 1 0.000 0 0 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T01:57:52.015 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 24.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???) e1 preinit fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???).mds e1 new map 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???).mds e1 print_map 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: e1 2024-04-27T01:57:52.016 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: -1 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout: No filesystems configured 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:51 smithi026 ceph-mon[21867]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:52 smithi026 bash[21848]: f821e7b12391c592a84a0d3aa3af4a9dbf5233ad2ab0b0b4ac7ad15468949875 2024-04-27T01:57:52.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:52 smithi026 systemd[1]: Started Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T01:57:52.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:52 smithi026 ceph-mon[21867]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-27T01:57:52.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:52 smithi026 ceph-mon[21867]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} 2024-04-27T01:57:52.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:52 smithi026 ceph-mon[21867]: fsmap 2024-04-27T01:57:52.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:52 smithi026 ceph-mon[21867]: osdmap e1: 0 total, 0 up, 0 in 2024-04-27T01:57:52.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:52 smithi026 ceph-mon[21867]: mgrmap e1: no daemons active 2024-04-27T01:57:53.027 INFO:teuthology.orchestra.run.smithi026.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-27T01:57:53.027 INFO:teuthology.orchestra.run.smithi026.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-27T01:57:53.028 INFO:teuthology.orchestra.run.smithi026.stderr:Creating mgr... 2024-04-27T01:57:53.028 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying port 9283 ... 2024-04-27T01:57:53.204 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to reset failed state of unit ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-27T01:57:53.212 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/ceph-7a5462be-0439-11ef-bc93-c7b262605968.target.wants/ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-7a5462be-0439-11ef-bc93-c7b262605968@.service. 2024-04-27T01:57:53.712 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:53 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1442679055' entity='client.admin' 2024-04-27T01:57:53.883 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-27T01:57:53.891 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-27T01:57:53.892 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-27T01:57:53.892 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to enable service . firewalld.service is not available 2024-04-27T01:57:53.902 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-27T01:57:53.910 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-27T01:57:53.912 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-27T01:57:53.912 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-27T01:57:53.912 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr to start... 2024-04-27T01:57:53.912 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr... 2024-04-27T01:57:53.979 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:53 smithi026 systemd[1]: Started Ceph mgr.y for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T01:57:54.233 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:53.998+0000 7f6d6fba8000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T01:57:54.233 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:54.136+0000 7f6d6fba8000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:57:54.233 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:54.241+0000 7f6d6fba8000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsid": "7a5462be-0439-11ef-bc93-c7b262605968", 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "health": { 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "checks": {}, 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mutes": [] 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum": [ 2024-04-27T01:57:54.497 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 0 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "a" 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "monmap": { 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osdmap": { 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-27T01:57:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgmap": { 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:54.499 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsmap": { 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": false, 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modules": [ 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "iostat", 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "nfs", 2024-04-27T01:57:54.500 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "restful" 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "servicemap": { 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modified": "2024-04-27T01:57:47.310820+0000", 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-27T01:57:54.501 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-27T01:57:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:54 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/542994179' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-27T01:57:54.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:54.638+0000 7f6d6fba8000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T01:57:54.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:54.723+0000 7f6d6fba8000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T01:57:55.052 INFO:teuthology.orchestra.run.smithi026.stderr:mgr not available, waiting (1/15)... 2024-04-27T01:57:56.152 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:55.919+0000 7f6d6fba8000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T01:57:56.153 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:56.087+0000 7f6d6fba8000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T01:57:56.153 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:56.171+0000 7f6d6fba8000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T01:57:56.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:56.492+0000 7f6d6fba8000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T01:57:56.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:56.584+0000 7f6d6fba8000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T01:57:56.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:56.679+0000 7f6d6fba8000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T01:57:57.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:56.927+0000 7f6d6fba8000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:57:57.498 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:57.393+0000 7f6d6fba8000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T01:57:57.503 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsid": "7a5462be-0439-11ef-bc93-c7b262605968", 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "health": { 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "checks": {}, 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mutes": [] 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum": [ 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 0 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "a" 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:57:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "monmap": { 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osdmap": { 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:57.505 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgmap": { 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsmap": { 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-27T01:57:57.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": false, 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modules": [ 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "iostat", 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "nfs", 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "restful" 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "servicemap": { 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modified": "2024-04-27T01:57:47.310820+0000", 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-27T01:57:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-27T01:57:58.099 INFO:teuthology.orchestra.run.smithi026.stderr:mgr not available, waiting (2/15)... 2024-04-27T01:57:58.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:57:57 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1066760797' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-27T01:57:58.171 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:57.906+0000 7f6d6fba8000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T01:57:58.171 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:58.002+0000 7f6d6fba8000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T01:57:58.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:58.308+0000 7f6d6fba8000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T01:57:58.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:58.399+0000 7f6d6fba8000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T01:57:59.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:58.822+0000 7f6d6fba8000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T01:57:59.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:58.958+0000 7f6d6fba8000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T01:57:59.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:59.053+0000 7f6d6fba8000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T01:57:59.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:59.170+0000 7f6d6fba8000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T01:57:59.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:59.260+0000 7f6d6fba8000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T01:58:00.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:57:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:57:59.851+0000 7f6d6fba8000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T01:58:00.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:00.233+0000 7f6d6fba8000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T01:58:00.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: Activating manager daemon y 2024-04-27T01:58:00.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: mgrmap e2: y(active, starting, since 0.00296657s) 2024-04-27T01:58:00.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T01:58:00.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: Manager daemon y is now available 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' 2024-04-27T01:58:00.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:00 smithi026 ceph-mon[21867]: from='mgr.14100 172.21.15.26:0/424153458' entity='mgr.y' 2024-04-27T01:58:02.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:02 smithi026 ceph-mon[21867]: mgrmap e3: y(active, since 1.00699s) 2024-04-27T01:58:04.099 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:58:04.099 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsid": "7a5462be-0439-11ef-bc93-c7b262605968", 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "health": { 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "checks": {}, 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mutes": [] 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum": [ 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 0 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "a" 2024-04-27T01:58:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "monmap": { 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osdmap": { 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-27T01:58:04.101 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgmap": { 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-27T01:58:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsmap": { 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": true, 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modules": [ 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "iostat", 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "nfs", 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "restful" 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-27T01:58:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "servicemap": { 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modified": "2024-04-27T01:57:47.310820+0000", 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-27T01:58:04.109 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-27T01:58:04.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:03 smithi026 ceph-mon[21867]: mgrmap e4: y(active, since 2s) 2024-04-27T01:58:04.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:03 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1079159096' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-27T01:58:09.513 INFO:teuthology.orchestra.run.smithi026.stderr:mgr is available 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [global] 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: fsid = 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [mgr] 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-27T01:58:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-27T01:58:11.868 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [osd] 2024-04-27T01:58:11.868 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-27T01:58:11.868 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-27T01:58:12.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:11 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1933520739' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-27T01:58:12.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:11 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1933520739' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-27T01:58:12.414 INFO:teuthology.orchestra.run.smithi026.stderr:Enabling cephadm module... 2024-04-27T01:58:13.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/558642093' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-27T01:58:13.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setuser ceph since I am not root 2024-04-27T01:58:13.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setgroup ceph since I am not root 2024-04-27T01:58:13.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:13.269+0000 7f17f246d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T01:58:13.675 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:13.405+0000 7f17f246d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:58:13.675 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:13.508+0000 7f17f246d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T01:58:13.942 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:13.835+0000 7f17f246d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T01:58:13.942 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:13.921+0000 7f17f246d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T01:58:14.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:14 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/558642093' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-27T01:58:14.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:14 smithi026 ceph-mon[21867]: mgrmap e5: y(active, since 12s) 2024-04-27T01:58:14.408 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-27T01:58:14.408 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-27T01:58:14.408 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": true, 2024-04-27T01:58:14.409 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-27T01:58:14.409 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-27T01:58:14.409 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-27T01:58:14.952 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for the mgr to restart... 2024-04-27T01:58:14.952 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr epoch 5... 2024-04-27T01:58:15.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:15 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1969508632' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-27T01:58:15.349 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:15.110+0000 7f17f246d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T01:58:15.349 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:15.280+0000 7f17f246d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T01:58:15.349 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:15.367+0000 7f17f246d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T01:58:16.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:15.695+0000 7f17f246d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T01:58:16.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:15.787+0000 7f17f246d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T01:58:16.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:15.883+0000 7f17f246d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T01:58:16.568 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:16.131+0000 7f17f246d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:58:16.568 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:16.587+0000 7f17f246d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T01:58:17.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:17 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:17.088+0000 7f17f246d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T01:58:17.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:17 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:17.184+0000 7f17f246d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T01:58:17.775 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:17 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:17.492+0000 7f17f246d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T01:58:17.775 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:17 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:17.586+0000 7f17f246d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T01:58:18.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:18.013+0000 7f17f246d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T01:58:18.439 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:18.151+0000 7f17f246d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T01:58:18.439 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:18.247+0000 7f17f246d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T01:58:18.439 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:18.365+0000 7f17f246d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T01:58:18.439 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:18.458+0000 7f17f246d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T01:58:19.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:19.027+0000 7f17f246d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T01:58:19.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:19.401+0000 7f17f246d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T01:58:20.446 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-27T01:58:20.447 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-27T01:58:20.447 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "initialized": true 2024-04-27T01:58:20.447 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-27T01:58:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: Active manager daemon y restarted 2024-04-27T01:58:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: Activating manager daemon y 2024-04-27T01:58:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: osdmap e2: 0 total, 0 up, 0 in 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: mgrmap e6: y(active, starting, since 0.0524776s) 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: Manager daemon y is now available 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T01:58:20.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:20 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T01:58:21.023 INFO:teuthology.orchestra.run.smithi026.stderr:mgr epoch 5 is available 2024-04-27T01:58:21.023 INFO:teuthology.orchestra.run.smithi026.stderr:Setting orchestrator backend to cephadm... 2024-04-27T01:58:21.273 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:01:58:21] ENGINE Bus STARTING 2024-04-27T01:58:21.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:01:58:21] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T01:58:21.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:01:58:21] ENGINE Bus STARTED 2024-04-27T01:58:21.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-mon[21867]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-27T01:58:21.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-mon[21867]: mgrmap e7: y(active, since 1.05471s) 2024-04-27T01:58:21.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-mon[21867]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-27T01:58:21.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:21.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:21 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:22.814 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: value unchanged 2024-04-27T01:58:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:22 smithi026 ceph-mon[21867]: [27/Apr/2024:01:58:21] ENGINE Bus STARTING 2024-04-27T01:58:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:22 smithi026 ceph-mon[21867]: [27/Apr/2024:01:58:21] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T01:58:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:22 smithi026 ceph-mon[21867]: [27/Apr/2024:01:58:21] ENGINE Bus STARTED 2024-04-27T01:58:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:22 smithi026 ceph-mon[21867]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:22 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:22 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:22 smithi026 ceph-mon[21867]: mgrmap e8: y(active, since 2s) 2024-04-27T01:58:23.336 INFO:teuthology.orchestra.run.smithi026.stderr:Generating ssh key... 2024-04-27T01:58:23.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:23 smithi026 ceph-mon[21867]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:24.597 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: Generating public/private rsa key pair. 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: Your identification has been saved in /tmp/tmp25t09lzl/key. 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: Your public key has been saved in /tmp/tmp25t09lzl/key.pub. 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: The key fingerprint is: 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: SHA256:AnvzKZB5wNOqmywQClEBbRteqBI7czyrq4RDWiW6IH4 ceph-7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: The key's randomart image is: 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: +---[RSA 3072]----+ 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |.+oo | 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |o =... | 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: | O.+* . | 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |O.*o O | 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |B*.o* = S | 2024-04-27T01:58:24.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |Xo.. + + . | 2024-04-27T01:58:24.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |*+.E . o | 2024-04-27T01:58:24.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |+o.o . | 2024-04-27T01:58:24.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: |+o+ | 2024-04-27T01:58:24.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: +----[SHA256]-----+ 2024-04-27T01:58:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-mon[21867]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-mon[21867]: Generating ssh key... 2024-04-27T01:58:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:24 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:25.495 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCouqFUf4ldhVgFke8zh3jNW1D9v7y+vMuly1U9fAwmPrcVWvWOCB7ty/Vy3OG6Sn5f88KbusV/QZNEqVQQxfpAEVmb2ov7nyKYAjkcHocaFnI2u87+XKwuJS52+zwljvNtnjULVLIdzHi1Pc+kGQZJDbXcXrCSQc1Amk2fUGMVtZ1PqUhTZkulbBZOvy78Grn8hMQ2y1+afhdAfxMwu19ZEx5CO7TLAfRDdn1cLm0t0O1FZc2kCtXhkvir11TeMBdSw4AJCrgAJgVcZ5oZernSqus2xTG6YrmHTdk21NAtE+vSd09wTcPUu6h3NEJEl0XT9XFKr+DJlJf1Ck+nolLMqMhldLZ4IebBuPNshi7j+PXL4vjuwYd2wolQufxCKVfgZdsGxSrUuLwJpjfReDMcuDb2WmIGC2YdBkpifeN7zzAPWX3l8EL2LV8sCFNk/T/YrTOFxCYSxQRw/lxpwZVoL96V1MWj4cX2LSdxfTePmknAvm5sWFOl5Iozw9WxSU= ceph-7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:58:25.965 INFO:teuthology.orchestra.run.smithi026.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-27T01:58:25.965 INFO:teuthology.orchestra.run.smithi026.stderr:Adding key to root@localhost authorized_keys... 2024-04-27T01:58:25.966 INFO:teuthology.orchestra.run.smithi026.stderr:Adding host smithi026... 2024-04-27T01:58:26.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:26 smithi026 ceph-mon[21867]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:27.435 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Added host 'smithi026' with addr '172.21.15.26' 2024-04-27T01:58:28.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:27 smithi026 ceph-mon[21867]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi026", "addr": "172.21.15.26", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:28.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:27 smithi026 ceph-mon[21867]: Deploying cephadm binary to smithi026 2024-04-27T01:58:28.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:27 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:28.011 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:27 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:28.014 INFO:teuthology.orchestra.run.smithi026.stderr:Deploying unmanaged mon service... 2024-04-27T01:58:28.643 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-27T01:58:29.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:28 smithi026 ceph-mon[21867]: Added host smithi026 2024-04-27T01:58:29.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:28 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:29.371 INFO:teuthology.orchestra.run.smithi026.stderr:Deploying unmanaged mgr service... 2024-04-27T01:58:30.059 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-27T01:58:30.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:29 smithi026 ceph-mon[21867]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:30.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:29 smithi026 ceph-mon[21867]: Saving service mon spec with placement count:5 2024-04-27T01:58:31.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:31 smithi026 ceph-mon[21867]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:31.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:31 smithi026 ceph-mon[21867]: Saving service mgr spec with placement count:2 2024-04-27T01:58:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:31 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:32.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:32 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:32.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:32 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:33.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:33 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3945491304' entity='client.admin' 2024-04-27T01:58:33.788 INFO:teuthology.orchestra.run.smithi026.stderr:Enabling the dashboard module... 2024-04-27T01:58:34.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:34 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4215378510' entity='client.admin' 2024-04-27T01:58:35.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/542448993' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-27T01:58:35.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:35.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-mon[21867]: from='mgr.14118 172.21.15.26:0/3802749571' entity='mgr.y' 2024-04-27T01:58:35.577 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setuser ceph since I am not root 2024-04-27T01:58:35.578 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setgroup ceph since I am not root 2024-04-27T01:58:35.578 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:35.466+0000 7f451365b000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T01:58:35.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:35.607+0000 7f451365b000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:58:35.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:35.708+0000 7f451365b000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T01:58:36.230 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:36.030+0000 7f451365b000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T01:58:36.230 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:36.116+0000 7f451365b000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T01:58:36.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:36 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/542448993' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-27T01:58:36.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:36 smithi026 ceph-mon[21867]: mgrmap e9: y(active, since 15s) 2024-04-27T01:58:36.628 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-27T01:58:36.628 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-27T01:58:36.629 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": true, 2024-04-27T01:58:36.629 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-27T01:58:36.629 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-27T01:58:36.629 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-27T01:58:37.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:37 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3227152030' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-27T01:58:37.493 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:37 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:37.339+0000 7f451365b000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T01:58:37.493 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:37 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:37.511+0000 7f451365b000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T01:58:37.805 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:37 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:37.597+0000 7f451365b000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T01:58:37.808 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for the mgr to restart... 2024-04-27T01:58:37.808 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr epoch 9... 2024-04-27T01:58:38.058 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:37 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:37.931+0000 7f451365b000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T01:58:38.058 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:38 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:38.026+0000 7f451365b000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T01:58:38.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:38 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:38.125+0000 7f451365b000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T01:58:38.822 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:38 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:38.378+0000 7f451365b000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:58:38.822 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:38 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:38.840+0000 7f451365b000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T01:58:39.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:39 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:39.341+0000 7f451365b000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T01:58:39.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:39 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:39.438+0000 7f451365b000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T01:58:40.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:39 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:39.745+0000 7f451365b000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T01:58:40.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:39 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:39.837+0000 7f451365b000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T01:58:40.585 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:40.253+0000 7f451365b000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T01:58:40.585 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:40.390+0000 7f451365b000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T01:58:40.586 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:40.485+0000 7f451365b000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T01:58:40.586 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:40.603+0000 7f451365b000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T01:58:40.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:40.695+0000 7f451365b000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T01:58:41.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:41.265+0000 7f451365b000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T01:58:42.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T01:58:41.637+0000 7f451365b000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T01:58:42.583 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:01:58:42] ENGINE Bus STARTING 2024-04-27T01:58:42.583 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:01:58:42] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T01:58:42.683 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-27T01:58:42.685 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-27T01:58:42.685 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "initialized": true 2024-04-27T01:58:42.685 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-27T01:58:42.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:01:58:42] ENGINE Bus STARTED 2024-04-27T01:58:42.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: Active manager daemon y restarted 2024-04-27T01:58:42.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: Activating manager daemon y 2024-04-27T01:58:42.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: osdmap e3: 0 total, 0 up, 0 in 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: mgrmap e10: y(active, starting, since 0.0527087s) 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: Manager daemon y is now available 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T01:58:42.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:43.310 INFO:teuthology.orchestra.run.smithi026.stderr:mgr epoch 9 is available 2024-04-27T01:58:43.310 INFO:teuthology.orchestra.run.smithi026.stderr:Generating a dashboard self-signed certificate... 2024-04-27T01:58:44.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:43 smithi026 ceph-mon[21867]: [27/Apr/2024:01:58:42] ENGINE Bus STARTING 2024-04-27T01:58:44.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:43 smithi026 ceph-mon[21867]: [27/Apr/2024:01:58:42] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T01:58:44.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:43 smithi026 ceph-mon[21867]: [27/Apr/2024:01:58:42] ENGINE Bus STARTED 2024-04-27T01:58:44.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:43 smithi026 ceph-mon[21867]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-27T01:58:44.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:43 smithi026 ceph-mon[21867]: mgrmap e11: y(active, since 1.05553s) 2024-04-27T01:58:44.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:43 smithi026 ceph-mon[21867]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-27T01:58:44.091 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-27T01:58:44.616 INFO:teuthology.orchestra.run.smithi026.stderr:Creating initial admin user... 2024-04-27T01:58:44.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:44 smithi026 ceph-mon[21867]: mgrmap e12: y(active, since 2s) 2024-04-27T01:58:44.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:44.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:44 smithi026 ceph-mon[21867]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:44.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:44.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:45.552 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$jlNNdkT8lFSCPcuV6W/EfejAeIBHTcT8RPwL/hrCMM.lMuZrCOjuW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714183125, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-27T01:58:46.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:45 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:46.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:45 smithi026 ceph-mon[21867]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:46.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:45 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:46.360 INFO:teuthology.orchestra.run.smithi026.stderr:Fetching dashboard port number... 2024-04-27T01:58:47.005 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 8443 2024-04-27T01:58:47.815 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-27T01:58:47.824 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-27T01:58:47.824 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-27T01:58:47.825 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-27T01:58:47.827 INFO:teuthology.orchestra.run.smithi026.stderr:Ceph Dashboard is now available at: 2024-04-27T01:58:47.827 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:47.827 INFO:teuthology.orchestra.run.smithi026.stderr: URL: https://smithi026:8443/ 2024-04-27T01:58:47.827 INFO:teuthology.orchestra.run.smithi026.stderr: User: admin 2024-04-27T01:58:47.827 INFO:teuthology.orchestra.run.smithi026.stderr: Password: 8bgyzee7kl 2024-04-27T01:58:47.827 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:47.827 INFO:teuthology.orchestra.run.smithi026.stderr:Enabling autotune for osd_memory_target 2024-04-27T01:58:48.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:47 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1794534206' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-27T01:58:49.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:48 smithi026 ceph-mon[21867]: mgrmap e13: y(active, since 6s) 2024-04-27T01:58:49.843 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-27T01:58:50.459 INFO:teuthology.orchestra.run.smithi026.stderr:You can access the Ceph CLI with: 2024-04-27T01:58:50.460 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:50.460 INFO:teuthology.orchestra.run.smithi026.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-27T01:58:50.460 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:50.460 INFO:teuthology.orchestra.run.smithi026.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-27T01:58:50.460 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:50.461 INFO:teuthology.orchestra.run.smithi026.stderr: ceph telemetry on 2024-04-27T01:58:50.461 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:50.461 INFO:teuthology.orchestra.run.smithi026.stderr:For more information see: 2024-04-27T01:58:50.461 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:50.461 INFO:teuthology.orchestra.run.smithi026.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-27T01:58:50.461 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T01:58:50.461 INFO:teuthology.orchestra.run.smithi026.stderr:Bootstrap complete. 2024-04-27T01:58:50.497 INFO:tasks.cephadm:Fetching config... 2024-04-27T01:58:50.497 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:58:50.497 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-27T01:58:50.517 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-27T01:58:50.517 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:58:50.517 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-27T01:58:50.573 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-27T01:58:50.573 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:58:50.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-27T01:58:50.639 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-27T01:58:50.639 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:58:50.639 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-27T01:58:50.695 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-27T01:58:50.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCouqFUf4ldhVgFke8zh3jNW1D9v7y+vMuly1U9fAwmPrcVWvWOCB7ty/Vy3OG6Sn5f88KbusV/QZNEqVQQxfpAEVmb2ov7nyKYAjkcHocaFnI2u87+XKwuJS52+zwljvNtnjULVLIdzHi1Pc+kGQZJDbXcXrCSQc1Amk2fUGMVtZ1PqUhTZkulbBZOvy78Grn8hMQ2y1+afhdAfxMwu19ZEx5CO7TLAfRDdn1cLm0t0O1FZc2kCtXhkvir11TeMBdSw4AJCrgAJgVcZ5oZernSqus2xTG6YrmHTdk21NAtE+vSd09wTcPUu6h3NEJEl0XT9XFKr+DJlJf1Ck+nolLMqMhldLZ4IebBuPNshi7j+PXL4vjuwYd2wolQufxCKVfgZdsGxSrUuLwJpjfReDMcuDb2WmIGC2YdBkpifeN7zzAPWX3l8EL2LV8sCFNk/T/YrTOFxCYSxQRw/lxpwZVoL96V1MWj4cX2LSdxfTePmknAvm5sWFOl5Iozw9WxSU= ceph-7a5462be-0439-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-27T01:58:50.770 INFO:teuthology.orchestra.run.smithi026.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCouqFUf4ldhVgFke8zh3jNW1D9v7y+vMuly1U9fAwmPrcVWvWOCB7ty/Vy3OG6Sn5f88KbusV/QZNEqVQQxfpAEVmb2ov7nyKYAjkcHocaFnI2u87+XKwuJS52+zwljvNtnjULVLIdzHi1Pc+kGQZJDbXcXrCSQc1Amk2fUGMVtZ1PqUhTZkulbBZOvy78Grn8hMQ2y1+afhdAfxMwu19ZEx5CO7TLAfRDdn1cLm0t0O1FZc2kCtXhkvir11TeMBdSw4AJCrgAJgVcZ5oZernSqus2xTG6YrmHTdk21NAtE+vSd09wTcPUu6h3NEJEl0XT9XFKr+DJlJf1Ck+nolLMqMhldLZ4IebBuPNshi7j+PXL4vjuwYd2wolQufxCKVfgZdsGxSrUuLwJpjfReDMcuDb2WmIGC2YdBkpifeN7zzAPWX3l8EL2LV8sCFNk/T/YrTOFxCYSxQRw/lxpwZVoL96V1MWj4cX2LSdxfTePmknAvm5sWFOl5Iozw9WxSU= ceph-7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:58:50.782 DEBUG:teuthology.orchestra.run.smithi151:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCouqFUf4ldhVgFke8zh3jNW1D9v7y+vMuly1U9fAwmPrcVWvWOCB7ty/Vy3OG6Sn5f88KbusV/QZNEqVQQxfpAEVmb2ov7nyKYAjkcHocaFnI2u87+XKwuJS52+zwljvNtnjULVLIdzHi1Pc+kGQZJDbXcXrCSQc1Amk2fUGMVtZ1PqUhTZkulbBZOvy78Grn8hMQ2y1+afhdAfxMwu19ZEx5CO7TLAfRDdn1cLm0t0O1FZc2kCtXhkvir11TeMBdSw4AJCrgAJgVcZ5oZernSqus2xTG6YrmHTdk21NAtE+vSd09wTcPUu6h3NEJEl0XT9XFKr+DJlJf1Ck+nolLMqMhldLZ4IebBuPNshi7j+PXL4vjuwYd2wolQufxCKVfgZdsGxSrUuLwJpjfReDMcuDb2WmIGC2YdBkpifeN7zzAPWX3l8EL2LV8sCFNk/T/YrTOFxCYSxQRw/lxpwZVoL96V1MWj4cX2LSdxfTePmknAvm5sWFOl5Iozw9WxSU= ceph-7a5462be-0439-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-27T01:58:50.822 INFO:teuthology.orchestra.run.smithi151.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCouqFUf4ldhVgFke8zh3jNW1D9v7y+vMuly1U9fAwmPrcVWvWOCB7ty/Vy3OG6Sn5f88KbusV/QZNEqVQQxfpAEVmb2ov7nyKYAjkcHocaFnI2u87+XKwuJS52+zwljvNtnjULVLIdzHi1Pc+kGQZJDbXcXrCSQc1Amk2fUGMVtZ1PqUhTZkulbBZOvy78Grn8hMQ2y1+afhdAfxMwu19ZEx5CO7TLAfRDdn1cLm0t0O1FZc2kCtXhkvir11TeMBdSw4AJCrgAJgVcZ5oZernSqus2xTG6YrmHTdk21NAtE+vSd09wTcPUu6h3NEJEl0XT9XFKr+DJlJf1Ck+nolLMqMhldLZ4IebBuPNshi7j+PXL4vjuwYd2wolQufxCKVfgZdsGxSrUuLwJpjfReDMcuDb2WmIGC2YdBkpifeN7zzAPWX3l8EL2LV8sCFNk/T/YrTOFxCYSxQRw/lxpwZVoL96V1MWj4cX2LSdxfTePmknAvm5sWFOl5Iozw9WxSU= ceph-7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:58:50.833 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-27T01:58:51.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:50 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/328191554' entity='client.admin' 2024-04-27T01:58:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:54 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2221130847' entity='client.admin' 2024-04-27T01:58:54.673 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-27T01:58:54.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-27T01:58:55.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:55 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:55.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:55 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi026", "name": "osd_memory_target"}]: dispatch 2024-04-27T01:58:55.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:55 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:55.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:55 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:55.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:55 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:57.181 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi151 2024-04-27T01:58:57.181 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-27T01:58:57.181 DEBUG:teuthology.orchestra.run.smithi151:> dd of=/etc/ceph/ceph.conf 2024-04-27T01:58:57.199 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-27T01:58:57.199 DEBUG:teuthology.orchestra.run.smithi151:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-27T01:58:57.256 INFO:tasks.cephadm:Adding host smithi151 to orchestrator... 2024-04-27T01:58:57.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch host add smithi151 2024-04-27T01:58:57.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:57.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:57 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:58:59.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:58:59 smithi026 ceph-mon[21867]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi151", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:00.151 INFO:teuthology.orchestra.run.smithi026.stdout:Added host 'smithi151' with addr '172.21.15.151' 2024-04-27T01:59:00.727 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-27T01:59:01.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: Deploying cephadm binary to smithi151 2024-04-27T01:59:01.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:01.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: Added host smithi151 2024-04-27T01:59:01.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:01.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:01.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:01.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:01.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:02.622 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T01:59:02.623 INFO:teuthology.orchestra.run.smithi026.stdout:[{"addr": "172.21.15.26", "hostname": "smithi026", "labels": [], "status": ""}, {"addr": "172.21.15.151", "hostname": "smithi151", "labels": [], "status": ""}] 2024-04-27T01:59:02.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:02 smithi026 ceph-mon[21867]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:02.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:02 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:03.189 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-27T01:59:03.189 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-27T01:59:03.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:03 smithi026 ceph-mon[21867]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-27T01:59:04.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:04 smithi026 ceph-mon[21867]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:04.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:04 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:04.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:04 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:04.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:04 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi151", "name": "osd_memory_target"}]: dispatch 2024-04-27T01:59:04.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:04 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:05.719 INFO:teuthology.orchestra.run.smithi026.stderr:adjusted tunables profile to default 2024-04-27T01:59:05.984 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:05 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1381100654' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-27T01:59:06.250 INFO:tasks.cephadm:Adding mon.a on smithi026 2024-04-27T01:59:06.251 INFO:tasks.cephadm:Adding mon.c on smithi026 2024-04-27T01:59:06.251 INFO:tasks.cephadm:Adding mon.b on smithi151 2024-04-27T01:59:06.251 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi151:172.21.15.151=b' 2024-04-27T01:59:07.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:06 smithi026 ceph-mon[21867]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:07.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:06 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1381100654' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-27T01:59:07.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:06 smithi026 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:08.019 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled mon update... 2024-04-27T01:59:08.667 DEBUG:teuthology.orchestra.run.smithi026:mon.c> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.c.service 2024-04-27T01:59:08.670 DEBUG:teuthology.orchestra.run.smithi151:mon.b> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.b.service 2024-04-27T01:59:08.672 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-27T01:59:08.672 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-27T01:59:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi151:172.21.15.151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi151:172.21.15.151=b;count:3 2024-04-27T01:59:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T01:59:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T01:59:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:10.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:10 smithi026 ceph-mon[21867]: Deploying daemon mon.c on smithi026 2024-04-27T01:59:10.524 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:59:10.524 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":1,"fsid":"7a5462be-0439-11ef-bc93-c7b262605968","modified":"2024-04-27T01:57:44.453030Z","created":"2024-04-27T01:57:44.453030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-27T01:59:10.525 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 1 2024-04-27T01:59:11.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:11 smithi026 ceph-mon[21867]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:11.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:11 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/3487014873' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:12.120 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-27T01:59:12.120 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-27T01:59:16.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: mon.a calling monitor election 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: mon.c calling monitor election 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-27T01:59:16.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:16.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: fsmap 2024-04-27T01:59:16.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:16.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: mgrmap e13: y(active, since 34s) 2024-04-27T01:59:16.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: overall HEALTH_OK 2024-04-27T01:59:16.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:16.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:16.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:17.147 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:59:17.147 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":2,"fsid":"7a5462be-0439-11ef-bc93-c7b262605968","modified":"2024-04-27T01:59:11.417230Z","created":"2024-04-27T01:57:44.453030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-27T01:59:17.149 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 2 2024-04-27T01:59:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:17 smithi026 ceph-mon[21867]: Deploying daemon mon.b on smithi151 2024-04-27T01:59:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:17 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/2265955770' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:17 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:17 smithi026 ceph-mon[26568]: Deploying daemon mon.b on smithi151 2024-04-27T01:59:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:17 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/2265955770' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:17 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:18.902 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-27T01:59:18.902 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-27T01:59:19.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:18 smithi026 ceph-mon[21867]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:18 smithi026 ceph-mon[26568]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:20.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:20 smithi151 systemd[1]: Starting Ceph mon.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T01:59:20.822 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:20 smithi151 podman[22607]: 2024-04-27 01:59:20.579097934 +0000 UTC m=+0.123508498 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-27T01:59:20.823 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:20 smithi151 podman[22607]: 2024-04-27 01:59:20.841590322 +0000 UTC m=+0.386000885 container create 8dcb4692da8749083ac35d6c8d7f2b01df146c28b6e9d6155e5051b72d5f4a70 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, ceph=True, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, release=754, RELEASE=HEAD, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-27T01:59:20.907 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:59:20.907 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":2,"fsid":"7a5462be-0439-11ef-bc93-c7b262605968","modified":"2024-04-27T01:59:11.417230Z","created":"2024-04-27T01:57:44.453030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-27T01:59:20.909 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 2 2024-04-27T01:59:21.352 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 podman[22607]: 2024-04-27 01:59:21.277924283 +0000 UTC m=+0.822334846 container init 8dcb4692da8749083ac35d6c8d7f2b01df146c28b6e9d6155e5051b72d5f4a70 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-27T01:59:21.352 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 podman[22607]: 2024-04-27 01:59:21.28132172 +0000 UTC m=+0.825732283 container start 8dcb4692da8749083ac35d6c8d7f2b01df146c28b6e9d6155e5051b72d5f4a70 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, distribution-scope=public, ceph=True, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, name=centos-stream, version=8, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True) 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: set uid:gid to 167:167 (ceph:ceph) 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: pidfile_write: ignore empty --pid-file 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: load: jerasure load: lrc 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: RocksDB version: 6.15.5 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Compile date Apr 18 2022 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: DB SUMMARY 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: DB Session ID: NJAI8IAJVTCD4APWE8TS 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: CURRENT file: CURRENT 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: IDENTITY file: IDENTITY 2024-04-27T01:59:21.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-27T01:59:21.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-27T01:59:21.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-27T01:59:21.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.error_if_exists: 0 2024-04-27T01:59:21.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.create_if_missing: 0 2024-04-27T01:59:21.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.paranoid_checks: 1 2024-04-27T01:59:21.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.env: 0x55f79eac6860 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.fs: Posix File System 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.info_log: 0x55f7a1115dc0 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_file_opening_threads: 16 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.statistics: (nil) 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.use_fsync: 0 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_log_file_size: 0 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-27T01:59:21.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.keep_log_file_num: 1000 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.recycle_log_file_num: 0 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.allow_fallocate: 1 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.allow_mmap_reads: 0 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.allow_mmap_writes: 0 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.use_direct_reads: 0 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.create_missing_column_families: 0 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.db_log_dir: 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-27T01:59:21.356 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-27T01:59:21.357 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-27T01:59:21.357 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-27T01:59:21.357 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-27T01:59:21.357 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-27T01:59:21.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.advise_random_on_open: 1 2024-04-27T01:59:21.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.db_write_buffer_size: 0 2024-04-27T01:59:21.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.write_buffer_manager: 0x55f7a1206240 2024-04-27T01:59:21.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-27T01:59:21.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-27T01:59:21.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-27T01:59:21.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.rate_limiter: (nil) 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.wal_recovery_mode: 2 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.enable_thread_tracking: 0 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.enable_pipelined_write: 0 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.unordered_write: 0 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.row_cache: None 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.wal_filter: None 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-27T01:59:21.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.allow_ingest_behind: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.preserve_deletes: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.two_write_queues: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.manual_wal_flush: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.atomic_flush: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.log_readahead_size: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.best_efforts_recovery: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.allow_data_in_errors: 0 2024-04-27T01:59:21.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.db_host_id: __hostname__ 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_background_jobs: 2 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_background_compactions: -1 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_subcompactions: 1 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_total_wal_size: 0 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_open_files: -1 2024-04-27T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bytes_per_sync: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_readahead_size: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_background_flushes: -1 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Compression algorithms supported: 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kZSTD supported: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kXpressCompression supported: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kLZ4HCCompression supported: 1 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kLZ4Compression supported: 1 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kBZip2Compression supported: 0 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kZlibCompression supported: 1 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: kSnappyCompression supported: 1 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-27T01:59:21.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.merge_operator: 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_filter: None 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_filter_factory: None 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.sst_partitioner_factory: None 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f7a10e1d00) 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks: 1 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_top_level_index_and_filter: 1 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_type: 0 2024-04-27T01:59:21.364 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_index_type: 0 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_shortening: 1 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: hash_index_allow_collision: 1 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: checksum: 1 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: no_block_cache: 0 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache: 0x55f7a114c170 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_name: BinnedLRUCache 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_options: 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: capacity : 536870912 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: num_shard_bits : 4 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: strict_capacity_limit : 0 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: high_pri_pool_ratio: 0.000 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_compressed: (nil) 2024-04-27T01:59:21.365 INFO:journalctl@ceph.mon.b.smithi151.stdout: persistent_cache: (nil) 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size: 4096 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size_deviation: 10 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_restart_interval: 16 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_block_restart_interval: 1 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: metadata_block_size: 4096 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: partition_filters: 0 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: use_delta_encoding: 1 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: whole_key_filtering: 1 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: verify_compression: 0 2024-04-27T01:59:21.366 INFO:journalctl@ceph.mon.b.smithi151.stdout: read_amp_bytes_per_bit: 0 2024-04-27T01:59:21.369 INFO:journalctl@ceph.mon.b.smithi151.stdout: format_version: 4 2024-04-27T01:59:21.369 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_index_compression: 1 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_align: 0 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.write_buffer_size: 33554432 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_write_buffer_number: 2 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression: NoCompression 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression: Disabled 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.prefix_extractor: nullptr 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.num_levels: 7 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-27T01:59:21.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression_opts.level: 32767 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression_opts.strategy: 0 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compression_opts.enabled: false 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-27T01:59:21.371 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.target_file_size_base: 67108864 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-27T01:59:21.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.arena_block_size: 4194304 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.disable_auto_compactions: 0 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-27T01:59:21.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.table_properties_collectors: 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.inplace_update_support: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.bloom_locality: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.max_successive_merges: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.paranoid_file_checks: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.force_consistency_checks: 1 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.report_bg_io_stats: 0 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.ttl: 2592000 2024-04-27T01:59:21.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.enable_blob_files: false 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.min_blob_size: 0 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.blob_file_size: 268435456 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714183161309013, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714183161309459, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714183161, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "d4f06554-cb1b-4170-8189-728f801b1f7e", "db_session_id": "NJAI8IAJVTCD4APWE8TS"}} 2024-04-27T01:59:21.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714183161309868, "job": 1, "event": "recovery_finished"} 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f7a1132a80 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: DB pointer 0x55f7a1142000 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** DB Stats ** 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-27T01:59:21.376 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T01:59:21.377 INFO:journalctl@ceph.mon.b.smithi151.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:59:21.378 INFO:journalctl@ceph.mon.b.smithi151.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-27T01:59:21.379 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: using public_addr v2:172.21.15.151:0/0 -> [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: starting mon.b rank -1 at public addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] at bind addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(???) e0 preinit fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).mds e1 new map 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: e1 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: legacy client fscid: -1 2024-04-27T01:59:21.380 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout: No filesystems configured 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/3487014873' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:59:21.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.a calling monitor election 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.c calling monitor election 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:21.382 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: fsmap 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mgrmap e13: y(active, since 34s) 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: overall HEALTH_OK 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: Deploying daemon mon.b on smithi151 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/2265955770' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/1699073299' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:21.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-27T01:59:21.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 ceph-mon[22663]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-27T01:59:21.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 bash[22607]: 8dcb4692da8749083ac35d6c8d7f2b01df146c28b6e9d6155e5051b72d5f4a70 2024-04-27T01:59:21.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:21 smithi151 systemd[1]: Started Ceph mon.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T01:59:21.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:21 smithi026 ceph-mon[21867]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:21 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/1699073299' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:21.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:21 smithi026 ceph-mon[26568]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:21.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:21 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/1699073299' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:22.735 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-27T01:59:22.735 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-27T01:59:26.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: mon.a calling monitor election 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: mon.c calling monitor election 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:26.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: fsmap 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: mgrmap e13: y(active, since 44s) 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: mon.b (rank 2) addr [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] is down (out of quorum) 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:59:26.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: mon.a calling monitor election 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: mon.c calling monitor election 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: fsmap 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: mgrmap e13: y(active, since 44s) 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: mon.b (rank 2) addr [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] is down (out of quorum) 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:26.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:26 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-27T01:59:27.485 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":3,"fsid":"7a5462be-0439-11ef-bc93-c7b262605968","modified":"2024-04-27T01:59:21.330741Z","created":"2024-04-27T01:57:44.453030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-27T01:59:27.487 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 3 2024-04-27T01:59:27.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T01:59:27.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:27.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/4163471422' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/4163471422' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:27.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:27.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:27.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:27.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:28.064 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-27T01:59:28.064 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-27T01:59:28.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T01:59:28.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.a calling monitor election 2024-04-27T01:59:28.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.c calling monitor election 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: fsmap 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mgrmap e13: y(active, since 44s) 2024-04-27T01:59:28.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.b (rank 2) addr [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] is down (out of quorum) 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:28.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:28.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/4163471422' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-27T01:59:28.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:28.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:28.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:28.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:28.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: mon.b calling monitor election 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: Reconfiguring mon.a (unknown last config time)... 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: Reconfiguring daemon mon.a on smithi026 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: mon.b calling monitor election 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: mon.c calling monitor election 2024-04-27T01:59:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: mon.a calling monitor election 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: fsmap 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: mgrmap e13: y(active, since 46s) 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: Cluster is now healthy 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[26568]: overall HEALTH_OK 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: mon.b calling monitor election 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T01:59:28.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: Reconfiguring mon.a (unknown last config time)... 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: Reconfiguring daemon mon.a on smithi026 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: mon.b calling monitor election 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: mon.c calling monitor election 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: mon.a calling monitor election 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: fsmap 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: mgrmap e13: y(active, since 46s) 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: Cluster is now healthy 2024-04-27T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:28 smithi026 ceph-mon[21867]: overall HEALTH_OK 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.b calling monitor election 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Reconfiguring mon.a (unknown last config time)... 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Reconfiguring daemon mon.a on smithi026 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.b calling monitor election 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.c calling monitor election 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.a calling monitor election 2024-04-27T01:59:29.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T01:59:29.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T01:59:29.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: fsmap 2024-04-27T01:59:29.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: osdmap e4: 0 total, 0 up, 0 in 2024-04-27T01:59:29.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: mgrmap e13: y(active, since 46s) 2024-04-27T01:59:29.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-27T01:59:29.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: Cluster is now healthy 2024-04-27T01:59:29.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:28 smithi151 ceph-mon[22663]: overall HEALTH_OK 2024-04-27T01:59:29.814 INFO:teuthology.orchestra.run.smithi026.stdout:# minimal ceph.conf for 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:59:29.815 INFO:teuthology.orchestra.run.smithi026.stdout:[global] 2024-04-27T01:59:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: fsid = 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T01:59:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: mon_host = [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2024-04-27T01:59:29.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:29 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:29.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:29 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:30.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:29 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T01:59:30.509 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-27T01:59:30.509 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:59:30.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.conf 2024-04-27T01:59:30.535 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:59:30.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-27T01:59:30.602 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-27T01:59:30.602 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.conf 2024-04-27T01:59:30.631 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-27T01:59:30.632 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-27T01:59:30.698 INFO:tasks.cephadm:Adding mgr.y on smithi026 2024-04-27T01:59:30.699 INFO:tasks.cephadm:Adding mgr.x on smithi151 2024-04-27T01:59:30.699 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi026=y;smithi151=x' 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2421467653' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: Reconfiguring mon.c (monmap changed)... 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:30.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[21867]: Reconfiguring daemon mon.c on smithi026 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2421467653' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: Reconfiguring mon.c (monmap changed)... 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:30.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:30 smithi026 ceph-mon[26568]: Reconfiguring daemon mon.c on smithi026 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2421467653' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: Reconfiguring mon.c (monmap changed)... 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:31.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:30 smithi151 ceph-mon[22663]: Reconfiguring daemon mon.c on smithi026 2024-04-27T01:59:32.614 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled mgr update... 2024-04-27T01:59:32.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:32 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:32.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:32 smithi151 ceph-mon[22663]: Reconfiguring mon.b (monmap changed)... 2024-04-27T01:59:32.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:32 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:32.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:32 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:32.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:32 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:32.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:32 smithi151 ceph-mon[22663]: Reconfiguring daemon mon.b on smithi151 2024-04-27T01:59:32.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:32 smithi151 ceph-mon[22663]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[21867]: Reconfiguring mon.b (monmap changed)... 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[21867]: Reconfiguring daemon mon.b on smithi151 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[21867]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:32.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[26568]: Reconfiguring mon.b (monmap changed)... 2024-04-27T01:59:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T01:59:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T01:59:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[26568]: Reconfiguring daemon mon.b on smithi151 2024-04-27T01:59:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:32 smithi026 ceph-mon[26568]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:33.625 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:33 smithi151 ceph-mon[22663]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:33.626 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:33 smithi151 ceph-mon[22663]: Saving service mgr spec with placement smithi026=y;smithi151=x;count:2 2024-04-27T01:59:33.626 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:33 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:33.647 DEBUG:teuthology.orchestra.run.smithi151:mgr.x> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service 2024-04-27T01:59:33.650 INFO:tasks.cephadm:Deploying OSDs... 2024-04-27T01:59:33.650 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T01:59:33.650 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/scratch_devs of=/dev/stdout 2024-04-27T01:59:33.669 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-27T01:59:33.669 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_1 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-27 01:59:10.909441871 +0000 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-27 01:56:58.146829932 +0000 2024-04-27T01:59:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-27 01:56:58.146829932 +0000 2024-04-27T01:59:33.729 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-27 01:56:58.146829932 +0000 2024-04-27T01:59:33.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-27T01:59:33.797 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-27T01:59:33.797 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-27T01:59:33.797 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000127255 s, 4.0 MB/s 2024-04-27T01:59:33.798 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-27T01:59:33.858 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_2 2024-04-27T01:59:33.916 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-27 01:59:10.909441871 +0000 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-27 01:56:58.496824379 +0000 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-27 01:56:58.496824379 +0000 2024-04-27T01:59:33.917 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-27 01:56:58.496824379 +0000 2024-04-27T01:59:33.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-27T01:59:33.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:33 smithi026 ceph-mon[21867]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:33.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:33 smithi026 ceph-mon[21867]: Saving service mgr spec with placement smithi026=y;smithi151=x;count:2 2024-04-27T01:59:33.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:33 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:33.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:33 smithi026 ceph-mon[26568]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:33.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:33 smithi026 ceph-mon[26568]: Saving service mgr spec with placement smithi026=y;smithi151=x;count:2 2024-04-27T01:59:33.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:33 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:33.982 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-27T01:59:33.982 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-27T01:59:33.982 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000112014 s, 4.6 MB/s 2024-04-27T01:59:33.984 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-27T01:59:34.041 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_3 2024-04-27T01:59:34.099 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-27T01:59:34.099 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:34.099 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-27T01:59:34.099 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:34.099 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:34.099 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-27 01:59:10.910441855 +0000 2024-04-27T01:59:34.099 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-27 01:56:58.820819238 +0000 2024-04-27T01:59:34.100 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-27 01:56:58.820819238 +0000 2024-04-27T01:59:34.100 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-27 01:56:58.820819238 +0000 2024-04-27T01:59:34.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-27T01:59:34.164 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-27T01:59:34.164 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-27T01:59:34.164 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000117415 s, 4.4 MB/s 2024-04-27T01:59:34.166 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-27T01:59:34.223 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_4 2024-04-27T01:59:34.279 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-27T01:59:34.279 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:34.280 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-27T01:59:34.280 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:34.280 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:34.280 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-27 01:59:10.910441855 +0000 2024-04-27T01:59:34.280 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-27 01:56:59.154813939 +0000 2024-04-27T01:59:34.280 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-27 01:56:59.154813939 +0000 2024-04-27T01:59:34.280 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-27 01:56:59.154813939 +0000 2024-04-27T01:59:34.280 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-27T01:59:34.343 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-27T01:59:34.343 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-27T01:59:34.343 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000103671 s, 4.9 MB/s 2024-04-27T01:59:34.344 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-27T01:59:34.401 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-27T01:59:34.401 DEBUG:teuthology.orchestra.run.smithi151:> dd if=/scratch_devs of=/dev/stdout 2024-04-27T01:59:34.416 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-27T01:59:34.417 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_1 2024-04-27T01:59:34.471 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-27T01:59:34.471 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:34.471 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-27T01:59:34.471 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:34.471 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:34.472 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-27 01:59:20.014383316 +0000 2024-04-27T01:59:34.472 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-27 01:56:53.443678867 +0000 2024-04-27T01:59:34.472 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-27 01:56:53.443678867 +0000 2024-04-27T01:59:34.472 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-27 01:56:53.443678867 +0000 2024-04-27T01:59:34.472 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-27T01:59:34.535 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-27T01:59:34.535 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-27T01:59:34.535 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000263829 s, 1.9 MB/s 2024-04-27T01:59:34.536 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-27T01:59:34.592 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_2 2024-04-27T01:59:34.647 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-27T01:59:34.647 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:34.648 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-27T01:59:34.648 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:34.648 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:34.648 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-27 01:59:20.014383316 +0000 2024-04-27T01:59:34.648 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-27 01:56:53.796674598 +0000 2024-04-27T01:59:34.648 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-27 01:56:53.796674598 +0000 2024-04-27T01:59:34.648 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-27 01:56:53.796674598 +0000 2024-04-27T01:59:34.648 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-27T01:59:34.710 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-27T01:59:34.710 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-27T01:59:34.710 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000189666 s, 2.7 MB/s 2024-04-27T01:59:34.711 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-27T01:59:34.767 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_3 2024-04-27T01:59:34.822 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-27T01:59:34.822 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:34.822 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-27T01:59:34.823 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:34.823 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:34.823 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-27 01:59:20.015383303 +0000 2024-04-27T01:59:34.823 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-27 01:56:54.126670607 +0000 2024-04-27T01:59:34.823 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-27 01:56:54.126670607 +0000 2024-04-27T01:59:34.823 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-27 01:56:54.126670607 +0000 2024-04-27T01:59:34.823 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-27T01:59:34.884 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-27T01:59:34.884 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-27T01:59:34.885 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000254754 s, 2.0 MB/s 2024-04-27T01:59:34.886 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-27T01:59:34.941 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_4 2024-04-27T01:59:34.995 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-27T01:59:34.995 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-27T01:59:34.995 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-04-27T01:59:34.995 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-27T01:59:34.995 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-27T01:59:34.995 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-27 01:59:20.015383303 +0000 2024-04-27T01:59:34.996 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-27 01:56:54.470666446 +0000 2024-04-27T01:59:34.996 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-27 01:56:54.470666446 +0000 2024-04-27T01:59:34.996 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-27 01:56:54.470666446 +0000 2024-04-27T01:59:34.996 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-27T01:59:35.057 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-27T01:59:35.057 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-27T01:59:35.057 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.00020452 s, 2.5 MB/s 2024-04-27T01:59:35.059 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-27T01:59:35.113 INFO:tasks.cephadm:Deploying osd.0 on smithi026 with /dev/vg_nvme/lv_4... 2024-04-27T01:59:35.114 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-27T01:59:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:35.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T01:59:35.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-27T01:59:35.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T01:59:35.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:35.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:35 smithi151 ceph-mon[22663]: Deploying daemon mgr.x on smithi151 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[21867]: Deploying daemon mgr.x on smithi151 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:35.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:35 smithi026 ceph-mon[26568]: Deploying daemon mgr.x on smithi151 2024-04-27T01:59:36.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 podman[23829]: 2024-04-27 01:59:35.925741055 +0000 UTC m=+0.023007590 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-27T01:59:36.378 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 podman[23829]: 2024-04-27 01:59:36.221599463 +0000 UTC m=+0.318865985 container init b0459dc8195a588e31dcb378e7006290584232df0819af79ab6d116bc8e425f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.buildah.version=1.19.8, distribution-scope=public, vendor=Red Hat, Inc., ceph=True, architecture=x86_64, name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=) 2024-04-27T01:59:36.378 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 podman[23829]: 2024-04-27 01:59:36.224250892 +0000 UTC m=+0.321517405 container start b0459dc8195a588e31dcb378e7006290584232df0819af79ab6d116bc8e425f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, name=centos-stream, architecture=x86_64, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-04-27T01:59:36.378 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 ceph-mgr[23848]: -- 172.21.15.151:0/2311132922 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a11913c340 con 0x55a119114c00 2024-04-27T01:59:36.378 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 bash[23829]: b0459dc8195a588e31dcb378e7006290584232df0819af79ab6d116bc8e425f7 2024-04-27T01:59:36.379 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 systemd[1]: Started Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T01:59:36.379 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:36.375+0000 7f6185900000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T01:59:36.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:36.509+0000 7f6185900000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:59:36.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:36.619+0000 7f6185900000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T01:59:37.028 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-27T01:59:37.028 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-27T01:59:37.028 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2024-04-27T01:59:37.028 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2024-04-27T01:59:37.028 INFO:teuthology.orchestra.run.smithi026.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00844016 s, 1.2 GB/s 2024-04-27T01:59:37.028 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2024-04-27T01:59:37.280 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:36.951+0000 7f6185900000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T01:59:37.280 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:37.039+0000 7f6185900000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T01:59:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T01:59:37.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T01:59:37.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T01:59:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T01:59:37.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:37.651 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_4 2024-04-27T01:59:38.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-mon[22663]: Reconfiguring mgr.y (unknown last config time)... 2024-04-27T01:59:38.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-mon[22663]: Reconfiguring daemon mgr.y on smithi026 2024-04-27T01:59:38.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:38.183+0000 7f6185900000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T01:59:38.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:38.346+0000 7f6185900000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T01:59:38.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:38.429+0000 7f6185900000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T01:59:38.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:38 smithi026 ceph-mon[21867]: Reconfiguring mgr.y (unknown last config time)... 2024-04-27T01:59:38.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:38 smithi026 ceph-mon[21867]: Reconfiguring daemon mgr.y on smithi026 2024-04-27T01:59:38.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:38 smithi026 ceph-mon[26568]: Reconfiguring mgr.y (unknown last config time)... 2024-04-27T01:59:38.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:38 smithi026 ceph-mon[26568]: Reconfiguring daemon mgr.y on smithi026 2024-04-27T01:59:39.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:38.743+0000 7f6185900000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T01:59:39.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:38.832+0000 7f6185900000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T01:59:39.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:38.924+0000 7f6185900000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T01:59:39.583 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:39.165+0000 7f6185900000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T01:59:39.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-mon[22663]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:39.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:39.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:39.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:39.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:39.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[21867]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[26568]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:39.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:39.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:39.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:39.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:39.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:39 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:39.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:39.602+0000 7f6185900000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T01:59:40.335 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:40.087+0000 7f6185900000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T01:59:40.336 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:40.180+0000 7f6185900000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T01:59:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T01:59:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T01:59:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:40.474+0000 7f6185900000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T01:59:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:40.561+0000 7f6185900000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T01:59:40.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T01:59:40.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T01:59:40.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:40.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T01:59:40.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T01:59:40.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:41.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:40.961+0000 7f6185900000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T01:59:41.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:41.090+0000 7f6185900000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T01:59:41.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:41.183+0000 7f6185900000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T01:59:41.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:41.297+0000 7f6185900000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T01:59:41.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:41 smithi026 ceph-mon[21867]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:41.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:41 smithi026 ceph-mon[21867]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:41.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:41 smithi026 ceph-mon[26568]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:41.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:41 smithi026 ceph-mon[26568]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:41.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:41 smithi151 ceph-mon[22663]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:41.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:41 smithi151 ceph-mon[22663]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T01:59:41.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:41.386+0000 7f6185900000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T01:59:42.270 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:41.923+0000 7f6185900000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T01:59:42.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T01:59:42.289+0000 7f6185900000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T01:59:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T01:59:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T01:59:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: Standby manager daemon x started 2024-04-27T01:59:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T01:59:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T01:59:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T01:59:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:42 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T01:59:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T01:59:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: Standby manager daemon x started 2024-04-27T01:59:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T01:59:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T01:59:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T01:59:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:42 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1304178367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2062810488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]: dispatch 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]: dispatch 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]': finished 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: osdmap e5: 1 total, 0 up, 1 in 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1806180336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-27T01:59:43.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:43 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T01:59:43.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2062810488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]': finished 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: osdmap e5: 1 total, 0 up, 1 in 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1806180336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2062810488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b7a48d3-2b4e-4b09-a4e7-819387c92ab3"}]': finished 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: osdmap e5: 1 total, 0 up, 1 in 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1806180336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T01:59:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-27T01:59:43.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:43 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T01:59:44.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:44 smithi151 ceph-mon[22663]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:44.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:44 smithi026 ceph-mon[21867]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:44.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:44 smithi026 ceph-mon[26568]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:46.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:46 smithi026 ceph-mon[26568]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:46.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:46 smithi026 ceph-mon[21867]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:46 smithi151 ceph-mon[22663]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:49.048 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:49 smithi026 ceph-mon[21867]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:49.048 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:49 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T01:59:49.048 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:49 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:49.049 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:49 smithi026 ceph-mon[26568]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:49.049 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:49 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T01:59:49.049 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:49 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:49.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:49 smithi151 ceph-mon[22663]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:49.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:49 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T01:59:49.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:49 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:50.079 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:50 smithi026 ceph-mon[21867]: Deploying daemon osd.0 on smithi026 2024-04-27T01:59:50.079 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:50 smithi026 ceph-mon[26568]: Deploying daemon osd.0 on smithi026 2024-04-27T01:59:50.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:50 smithi151 ceph-mon[22663]: Deploying daemon osd.0 on smithi026 2024-04-27T01:59:51.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:51 smithi026 ceph-mon[21867]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:51.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:51 smithi026 ceph-mon[26568]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:51.252 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:51 smithi151 ceph-mon[22663]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:53.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:53 smithi026 ceph-mon[21867]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:53.151 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:53 smithi026 ceph-mon[26568]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:53.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:53 smithi151 ceph-mon[22663]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:54.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:54.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:54.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:54.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:54.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:54.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:54.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:54.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:55.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:55 smithi026 ceph-mon[21867]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:55.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:55 smithi026 ceph-mon[26568]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:55.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:55 smithi151 ceph-mon[22663]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:56.272 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 0 on host 'smithi026' 2024-04-27T01:59:56.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:56.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:56.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:56.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:56.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T01:59:56.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:56 smithi026 ceph-mon[26568]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T01:59:56.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:56.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T01:59:56.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T01:59:56.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T01:59:56.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T01:59:56.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T01:59:56.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:56 smithi151 ceph-mon[22663]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T01:59:56.895 DEBUG:teuthology.orchestra.run.smithi026:osd.0> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.0.service 2024-04-27T01:59:56.897 INFO:tasks.cephadm:Deploying osd.1 on smithi026 with /dev/vg_nvme/lv_3... 2024-04-27T01:59:56.897 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[26568]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[26568]: osdmap e6: 1 total, 0 up, 1 in 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[26568]: from='osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[26568]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[21867]: osdmap e6: 1 total, 0 up, 1 in 2024-04-27T01:59:58.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[21867]: from='osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T01:59:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T01:59:58.360 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[29440]: 2024-04-27T01:59:58.301+0000 7fa150d9f700 -1 osd.0 0 waiting for initial osdmap 2024-04-27T01:59:58.360 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 01:59:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[29440]: 2024-04-27T01:59:58.304+0000 7fa149f33700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T01:59:58.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:58 smithi151 ceph-mon[22663]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-27T01:59:58.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:58 smithi151 ceph-mon[22663]: osdmap e6: 1 total, 0 up, 1 in 2024-04-27T01:59:58.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:58 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:58.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:58 smithi151 ceph-mon[22663]: from='osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T01:59:58.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:58 smithi151 ceph-mon[22663]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T01:59:59.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[21867]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:59.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T01:59:59.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[21867]: osdmap e7: 1 total, 0 up, 1 in 2024-04-27T01:59:59.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:59.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:59.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[26568]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:59.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[26568]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T01:59:59.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[26568]: osdmap e7: 1 total, 0 up, 1 in 2024-04-27T01:59:59.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:59.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 01:59:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:59.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:59 smithi151 ceph-mon[22663]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-27T01:59:59.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:59 smithi151 ceph-mon[22663]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T01:59:59.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:59 smithi151 ceph-mon[22663]: osdmap e7: 1 total, 0 up, 1 in 2024-04-27T01:59:59.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T01:59:59.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 01:59:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:00:00.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:00:00.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:00:00.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[21867]: osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281] boot 2024-04-27T02:00:00.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[21867]: osdmap e8: 1 total, 1 up, 1 in 2024-04-27T02:00:00.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:00:00.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[21867]: overall HEALTH_OK 2024-04-27T02:00:00.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:00:00.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:00:00.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[26568]: osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281] boot 2024-04-27T02:00:00.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[26568]: osdmap e8: 1 total, 1 up, 1 in 2024-04-27T02:00:00.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:00:00.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:00 smithi026 ceph-mon[26568]: overall HEALTH_OK 2024-04-27T02:00:00.492 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-27T02:00:00.492 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-27T02:00:00.492 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2024-04-27T02:00:00.492 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2024-04-27T02:00:00.493 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00924262 s, 1.1 GB/s 2024-04-27T02:00:00.493 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2024-04-27T02:00:00.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:00 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:00:00.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:00 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:00:00.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:00 smithi151 ceph-mon[22663]: osd.0 [v2:172.21.15.26:6802/652787281,v1:172.21.15.26:6803/652787281] boot 2024-04-27T02:00:00.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:00 smithi151 ceph-mon[22663]: osdmap e8: 1 total, 1 up, 1 in 2024-04-27T02:00:00.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:00 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:00:00.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:00 smithi151 ceph-mon[22663]: overall HEALTH_OK 2024-04-27T02:00:01.175 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_3 2024-04-27T02:00:01.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[26568]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:01.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[26568]: osdmap e9: 1 total, 1 up, 1 in 2024-04-27T02:00:01.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:01.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[21867]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:01.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[21867]: osdmap e9: 1 total, 1 up, 1 in 2024-04-27T02:00:01.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:01.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:01 smithi151 ceph-mon[22663]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:01.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:01 smithi151 ceph-mon[22663]: osdmap e9: 1 total, 1 up, 1 in 2024-04-27T02:00:01.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:01 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:01 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:01.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:01 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:01.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:01 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:02.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:02 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi026 2024-04-27T02:00:02.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:02 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi026 to 16069M 2024-04-27T02:00:02.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:02 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi026 2024-04-27T02:00:02.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:02 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi026 to 16069M 2024-04-27T02:00:02.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:02 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi026 2024-04-27T02:00:02.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:02 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi026 to 16069M 2024-04-27T02:00:03.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[21867]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:03.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:03.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:03.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:03.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[26568]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:03.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:03.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:03.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:03 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:03.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:03 smithi151 ceph-mon[22663]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:03.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:03 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:03.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:03 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:03.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:03 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:04.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:04 smithi026 ceph-mon[21867]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:04.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:04 smithi026 ceph-mon[26568]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:04.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:04 smithi151 ceph-mon[22663]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:05.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:05 smithi151 ceph-mon[22663]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:05.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:05 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1850869079' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]: dispatch 2024-04-27T02:00:05.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:05 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]: dispatch 2024-04-27T02:00:05.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:05 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]': finished 2024-04-27T02:00:05.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:05 smithi151 ceph-mon[22663]: osdmap e10: 2 total, 1 up, 2 in 2024-04-27T02:00:05.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:05 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[21867]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1850869079' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]: dispatch 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]: dispatch 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]': finished 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[21867]: osdmap e10: 2 total, 1 up, 2 in 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[26568]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1850869079' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]: dispatch 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]: dispatch 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c9ce276b-a815-42f7-817b-2af147f9f597"}]': finished 2024-04-27T02:00:05.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[26568]: osdmap e10: 2 total, 1 up, 2 in 2024-04-27T02:00:05.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:05 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:06.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:06 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3403133402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:06.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:06 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3403133402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:06.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:06 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3403133402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:07.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:07 smithi151 ceph-mon[22663]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:07.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:07 smithi026 ceph-mon[26568]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:07.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:07 smithi026 ceph-mon[21867]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:09.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:09 smithi026 ceph-mon[21867]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:09.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:09 smithi026 ceph-mon[26568]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:09.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:09 smithi151 ceph-mon[22663]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:11.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:11 smithi026 ceph-mon[21867]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:11.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:11 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:00:11.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:11 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:11.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:11 smithi026 ceph-mon[26568]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:11.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:11 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:00:11.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:11 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:11.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:11 smithi151 ceph-mon[22663]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:11.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:11 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:00:11.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:11 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:12.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:12 smithi026 ceph-mon[21867]: Deploying daemon osd.1 on smithi026 2024-04-27T02:00:12.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:12 smithi026 ceph-mon[26568]: Deploying daemon osd.1 on smithi026 2024-04-27T02:00:12.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:12 smithi151 ceph-mon[22663]: Deploying daemon osd.1 on smithi026 2024-04-27T02:00:13.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:13 smithi151 ceph-mon[22663]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:13.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:13 smithi026 ceph-mon[21867]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:13.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:13 smithi026 ceph-mon[26568]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:15.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:15 smithi026 ceph-mon[21867]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:15.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:15 smithi026 ceph-mon[26568]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:15.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:15 smithi151 ceph-mon[22663]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[21867]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[26568]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:17.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:17.297 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:17.297 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:17 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:17.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:17 smithi151 ceph-mon[22663]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:17.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:17 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:17.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:17 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:17.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:17 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:17.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:17 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:19.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:19 smithi026 ceph-mon[21867]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:19.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:19 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:19.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:19 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:19.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:19 smithi026 ceph-mon[26568]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:19.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:19 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:19.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:19 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:19.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:19 smithi151 ceph-mon[22663]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:19.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:19 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:19.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:19 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:19.419 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 1 on host 'smithi026' 2024-04-27T02:00:20.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[26568]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[21867]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:20.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:20 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:20.049 DEBUG:teuthology.orchestra.run.smithi026:osd.1> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.1.service 2024-04-27T02:00:20.051 INFO:tasks.cephadm:Deploying osd.2 on smithi026 with /dev/vg_nvme/lv_2... 2024-04-27T02:00:20.051 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-27T02:00:20.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:20 smithi151 ceph-mon[22663]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-27T02:00:20.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:20 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:20.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:20 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:20.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:20 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:20.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:20 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:21.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[21867]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:21.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[21867]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-27T02:00:21.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[21867]: osdmap e11: 2 total, 1 up, 2 in 2024-04-27T02:00:21.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:21.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[21867]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:21.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[26568]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:21.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[26568]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-27T02:00:21.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[26568]: osdmap e11: 2 total, 1 up, 2 in 2024-04-27T02:00:21.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:21.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-mon[26568]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:21.247 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[32407]: 2024-04-27T02:00:21.046+0000 7fd11f6f0700 -1 osd.1 0 waiting for initial osdmap 2024-04-27T02:00:21.247 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[32407]: 2024-04-27T02:00:21.050+0000 7fd11a087700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:00:21.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:21 smithi151 ceph-mon[22663]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:21.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:21 smithi151 ceph-mon[22663]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-27T02:00:21.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:21 smithi151 ceph-mon[22663]: osdmap e11: 2 total, 1 up, 2 in 2024-04-27T02:00:21.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:21.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:21 smithi151 ceph-mon[22663]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:22.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:22 smithi151 ceph-mon[22663]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:00:22.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:22 smithi151 ceph-mon[22663]: osdmap e12: 2 total, 1 up, 2 in 2024-04-27T02:00:22.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:22 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:22.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:22 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[21867]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[21867]: osdmap e12: 2 total, 1 up, 2 in 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[26568]: from='osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[26568]: osdmap e12: 2 total, 1 up, 2 in 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:22.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:22 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:23.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:00:23.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[21867]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[21867]: osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840] boot 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[21867]: osdmap e13: 2 total, 2 up, 2 in 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[26568]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[26568]: osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840] boot 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[26568]: osdmap e13: 2 total, 2 up, 2 in 2024-04-27T02:00:23.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:23 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:23.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:23 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:00:23.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:23 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:00:23.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:23 smithi151 ceph-mon[22663]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-27T02:00:23.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:23 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:23.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:23 smithi151 ceph-mon[22663]: osd.1 [v2:172.21.15.26:6810/1040402840,v1:172.21.15.26:6811/1040402840] boot 2024-04-27T02:00:23.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:23 smithi151 ceph-mon[22663]: osdmap e13: 2 total, 2 up, 2 in 2024-04-27T02:00:23.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:23 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:00:23.672 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-27T02:00:23.672 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-27T02:00:23.672 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2024-04-27T02:00:23.672 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2024-04-27T02:00:23.672 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00978449 s, 1.1 GB/s 2024-04-27T02:00:23.672 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2024-04-27T02:00:24.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_2 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi026 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: osdmap e14: 2 total, 2 up, 2 in 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi026 to 8034M 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:24 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi026 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: osdmap e14: 2 total, 2 up, 2 in 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi026 to 8034M 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi026 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: osdmap e14: 2 total, 2 up, 2 in 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi026 to 8034M 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:24.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:24 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:25.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:25 smithi026 ceph-mon[26568]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:25.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:25 smithi026 ceph-mon[21867]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:25.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:25 smithi151 ceph-mon[22663]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:27.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:27 smithi151 ceph-mon[22663]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:27.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:27 smithi151 ceph-mon[22663]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:27.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:27 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:27.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:27 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:27.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:27 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[26568]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[26568]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[21867]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[21867]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:27.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:27.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:29.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:29 smithi151 ceph-mon[22663]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:29.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:29 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2048120697' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e22a75e4-af45-43bb-bf21-ab69c2046a4d"}]: dispatch 2024-04-27T02:00:29.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:29 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2048120697' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e22a75e4-af45-43bb-bf21-ab69c2046a4d"}]': finished 2024-04-27T02:00:29.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:29 smithi151 ceph-mon[22663]: osdmap e15: 3 total, 2 up, 3 in 2024-04-27T02:00:29.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:29 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:29.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:29 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1290024051' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[26568]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2048120697' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e22a75e4-af45-43bb-bf21-ab69c2046a4d"}]: dispatch 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2048120697' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e22a75e4-af45-43bb-bf21-ab69c2046a4d"}]': finished 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[26568]: osdmap e15: 3 total, 2 up, 3 in 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1290024051' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[21867]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2048120697' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e22a75e4-af45-43bb-bf21-ab69c2046a4d"}]: dispatch 2024-04-27T02:00:29.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2048120697' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e22a75e4-af45-43bb-bf21-ab69c2046a4d"}]': finished 2024-04-27T02:00:29.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[21867]: osdmap e15: 3 total, 2 up, 3 in 2024-04-27T02:00:29.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:29.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:29 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1290024051' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:31.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:31 smithi151 ceph-mon[22663]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:31.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:31 smithi026 ceph-mon[21867]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:31 smithi026 ceph-mon[26568]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:33.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:33 smithi026 ceph-mon[21867]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:33.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:33 smithi026 ceph-mon[26568]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:33.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:33 smithi151 ceph-mon[22663]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:35 smithi151 ceph-mon[22663]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:00:35.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:35 smithi026 ceph-mon[21867]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:00:35.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:35.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:35 smithi026 ceph-mon[26568]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:35.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:00:35.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:36.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:36 smithi026 ceph-mon[21867]: Deploying daemon osd.2 on smithi026 2024-04-27T02:00:36.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:36 smithi026 ceph-mon[26568]: Deploying daemon osd.2 on smithi026 2024-04-27T02:00:36.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:36 smithi151 ceph-mon[22663]: Deploying daemon osd.2 on smithi026 2024-04-27T02:00:37.340 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:37 smithi026 ceph-mon[21867]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:37.341 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:37 smithi026 ceph-mon[26568]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:37.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:37 smithi151 ceph-mon[22663]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:39.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:39 smithi026 ceph-mon[21867]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:39.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:39 smithi026 ceph-mon[26568]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:39.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:39 smithi151 ceph-mon[22663]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:40.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:40.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:40.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:40.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:40.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:40 smithi151 ceph-mon[22663]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[21867]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:40 smithi026 ceph-mon[26568]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:41.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:41 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:00:41.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:41 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:00:41.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:41 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:00:41.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:41 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:00:42.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:41 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:00:42.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:41 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:00:42.649 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 2 on host 'smithi026' 2024-04-27T02:00:43.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[21867]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:43.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:43.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[21867]: from='osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:00:43.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[21867]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:00:43.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[26568]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:43.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:43.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[26568]: from='osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:00:43.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:43 smithi026 ceph-mon[26568]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:00:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:43 smithi151 ceph-mon[22663]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:43 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:43 smithi151 ceph-mon[22663]: from='osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:00:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:43 smithi151 ceph-mon[22663]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:00:43.364 DEBUG:teuthology.orchestra.run.smithi026:osd.2> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.2.service 2024-04-27T02:00:43.366 INFO:tasks.cephadm:Deploying osd.3 on smithi026 with /dev/vg_nvme/lv_1... 2024-04-27T02:00:43.367 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-27T02:00:44.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:44.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[26568]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-27T02:00:44.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[26568]: osdmap e16: 3 total, 2 up, 3 in 2024-04-27T02:00:44.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:44.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[26568]: from='osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:44.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[26568]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:44.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[26568]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:44.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:44.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[21867]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-27T02:00:44.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[21867]: osdmap e16: 3 total, 2 up, 3 in 2024-04-27T02:00:44.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:44.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[21867]: from='osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:44.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[21867]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:44.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-mon[21867]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:44.778 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[35340]: 2024-04-27T02:00:44.674+0000 7f695ebd6700 -1 osd.2 0 waiting for initial osdmap 2024-04-27T02:00:44.778 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[35340]: 2024-04-27T02:00:44.677+0000 7f695a56f700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:00:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:44 smithi151 ceph-mon[22663]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-27T02:00:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:44 smithi151 ceph-mon[22663]: osdmap e16: 3 total, 2 up, 3 in 2024-04-27T02:00:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:44 smithi151 ceph-mon[22663]: from='osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:44.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:44 smithi151 ceph-mon[22663]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:00:44.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:44 smithi151 ceph-mon[22663]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-27T02:00:45.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[21867]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:00:45.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[21867]: osdmap e17: 3 total, 2 up, 3 in 2024-04-27T02:00:45.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:45.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:45.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[26568]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:00:45.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[26568]: osdmap e17: 3 total, 2 up, 3 in 2024-04-27T02:00:45.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:45.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:45 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:46.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:45 smithi151 ceph-mon[22663]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:00:46.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:45 smithi151 ceph-mon[22663]: osdmap e17: 3 total, 2 up, 3 in 2024-04-27T02:00:46.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:45 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:46.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:45 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:46.552 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-27T02:00:46.552 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-27T02:00:46.552 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2024-04-27T02:00:46.552 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2024-04-27T02:00:46.552 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0103117 s, 1.0 GB/s 2024-04-27T02:00:46.552 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2024-04-27T02:00:47.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:00:47.018 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:00:47.018 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[21867]: osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570] boot 2024-04-27T02:00:47.018 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[21867]: osdmap e18: 3 total, 3 up, 3 in 2024-04-27T02:00:47.018 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:47.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[21867]: pgmap v70: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:47.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-27T02:00:47.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:00:47.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:00:47.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[26568]: osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570] boot 2024-04-27T02:00:47.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[26568]: osdmap e18: 3 total, 3 up, 3 in 2024-04-27T02:00:47.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:47.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[26568]: pgmap v70: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:47.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:46 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-27T02:00:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:46 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:00:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:46 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:00:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:46 smithi151 ceph-mon[22663]: osd.2 [v2:172.21.15.26:6818/1403075570,v1:172.21.15.26:6819/1403075570] boot 2024-04-27T02:00:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:46 smithi151 ceph-mon[22663]: osdmap e18: 3 total, 3 up, 3 in 2024-04-27T02:00:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:46 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:00:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:46 smithi151 ceph-mon[22663]: pgmap v70: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:46 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-27T02:00:47.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_1 2024-04-27T02:00:47.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:47 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-27T02:00:47.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:47 smithi026 ceph-mon[26568]: osdmap e19: 3 total, 3 up, 3 in 2024-04-27T02:00:47.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:47 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-27T02:00:47.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:47 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-27T02:00:47.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:47 smithi026 ceph-mon[21867]: osdmap e19: 3 total, 3 up, 3 in 2024-04-27T02:00:47.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:47 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-27T02:00:48.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:47 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-27T02:00:48.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:47 smithi151 ceph-mon[22663]: osdmap e19: 3 total, 3 up, 3 in 2024-04-27T02:00:48.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:47 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-27T02:00:49.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-27T02:00:49.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: osdmap e20: 3 total, 3 up, 3 in 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi026 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi026 to 5356M 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:49.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: osdmap e20: 3 total, 3 up, 3 in 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi026 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi026 to 5356M 2024-04-27T02:00:49.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:49.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:49.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:49.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:48 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: osdmap e20: 3 total, 3 up, 3 in 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi026 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:00:49.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi026 to 5356M 2024-04-27T02:00:49.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:00:49.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:49.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:00:49.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:48 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:49.291 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:00:49 smithi026 sudo[36651]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-27T02:00:49.291 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:00:49 smithi026 sudo[36651]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:49.291 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:00:49 smithi026 sudo[36651]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:49.859 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:00:49 smithi026 sudo[36689]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-27T02:00:49.859 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:00:49 smithi026 sudo[36689]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:49.859 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:00:49 smithi026 sudo[36689]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:50.273 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36759]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-27T02:00:50.273 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36759]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:50.273 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36759]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:50.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[21867]: osdmap e21: 3 total, 3 up, 3 in 2024-04-27T02:00:50.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:50.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:50.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:50.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:50.274 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[26568]: osdmap e21: 3 total, 3 up, 3 in 2024-04-27T02:00:50.274 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:50.274 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:50.274 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:50.274 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:50 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:50.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:50 smithi151 ceph-mon[22663]: osdmap e21: 3 total, 3 up, 3 in 2024-04-27T02:00:50.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:50 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:50.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:50 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:00:50.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:50 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:00:50.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:50 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:00:50.608 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36792]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-27T02:00:50.609 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36792]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:50.609 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36792]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:51.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:51 smithi026 ceph-mon[26568]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:51.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:51 smithi026 ceph-mon[26568]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:51.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:51 smithi026 ceph-mon[26568]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-27T02:00:51.248 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36799]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-27T02:00:51.248 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36799]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:51.249 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:50 smithi026 sudo[36799]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:51.249 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:51 smithi026 ceph-mon[21867]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:51.249 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:51 smithi026 ceph-mon[21867]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:51.249 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:51 smithi026 ceph-mon[21867]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-27T02:00:51.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:51 smithi151 ceph-mon[22663]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:51.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:51 smithi151 ceph-mon[22663]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:00:51.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:51 smithi151 ceph-mon[22663]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-27T02:00:51.517 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:51 smithi026 sudo[36825]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-27T02:00:51.518 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:51 smithi026 sudo[36825]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:51.518 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:00:51 smithi026 sudo[36825]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:52.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:51 smithi026 sudo[36857]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-27T02:00:52.322 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:51 smithi026 sudo[36857]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:52.322 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:51 smithi026 sudo[36857]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:52.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:52 smithi026 sudo[36867]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-27T02:00:52.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:52 smithi026 sudo[36867]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:52.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:52 smithi026 sudo[36867]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:53.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:52 smithi026 sudo[36902]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-27T02:00:53.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:52 smithi026 sudo[36902]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:53.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:52 smithi026 sudo[36902]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:53.488 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/600439581' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]': finished 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: osdmap e22: 4 total, 3 up, 4 in 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:53.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 sudo[24187]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-27T02:00:53.490 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 sudo[24187]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:53.490 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 sudo[24187]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/600439581' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]: dispatch 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]: dispatch 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]': finished 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: osdmap e22: 4 total, 3 up, 4 in 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 sudo[36905]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 sudo[36905]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 sudo[36905]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:53.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/600439581' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a048a9f8-c1a3-4da7-908c-175069e4e4cd"}]': finished 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: osdmap e22: 4 total, 3 up, 4 in 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:53.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:53.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 sudo[24190]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-27T02:00:53.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 sudo[24190]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-27T02:00:53.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:53 smithi151 sudo[24190]: pam_unix(sudo:session): session closed for user root 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2257723263' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:54.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:54.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:54.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2257723263' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:00:54.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:00:54.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:00:54.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2257723263' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:00:54.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-27T02:00:54.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:00:55.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:55 smithi151 ceph-mon[22663]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:55.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:55 smithi026 ceph-mon[21867]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:55.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:55 smithi026 ceph-mon[26568]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:57.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:57 smithi026 ceph-mon[21867]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:57.510 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:57 smithi026 ceph-mon[26568]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:57.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:57 smithi151 ceph-mon[22663]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:59.495 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:00:59 smithi026 ceph-mon[21867]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:59.495 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:00:59 smithi026 ceph-mon[26568]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:00:59.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:00:59 smithi151 ceph-mon[22663]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:01:00.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:00 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:01:00.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:00 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:00.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:00 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:01:00.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:00 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:00.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:00 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:01:00.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:00 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:01.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:01 smithi026 ceph-mon[21867]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-27T02:01:01.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:01 smithi026 ceph-mon[21867]: Deploying daemon osd.3 on smithi026 2024-04-27T02:01:01.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:01 smithi026 ceph-mon[26568]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-27T02:01:01.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:01 smithi026 ceph-mon[26568]: Deploying daemon osd.3 on smithi026 2024-04-27T02:01:01.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:01 smithi151 ceph-mon[22663]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-27T02:01:01.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:01 smithi151 ceph-mon[22663]: Deploying daemon osd.3 on smithi026 2024-04-27T02:01:03.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:03 smithi151 ceph-mon[22663]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-27T02:01:03.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:03 smithi026 ceph-mon[21867]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-27T02:01:03.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:03 smithi026 ceph-mon[26568]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[21867]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[26568]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:05.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:05.549 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:05.549 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:05 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:05.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:05 smithi151 ceph-mon[22663]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-27T02:01:05.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:05 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:05.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:05 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:05.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:05 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:05.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:05 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:07.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:07 smithi151 ceph-mon[22663]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-27T02:01:07.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:07 smithi026 ceph-mon[21867]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-27T02:01:07.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:07 smithi026 ceph-mon[26568]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-27T02:01:08.457 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 3 on host 'smithi026' 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:09.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:08 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:09.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:09.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:08 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:09.112 DEBUG:teuthology.orchestra.run.smithi026:osd.3> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.3.service 2024-04-27T02:01:09.115 INFO:tasks.cephadm:Deploying osd.4 on smithi151 with /dev/vg_nvme/lv_4... 2024-04-27T02:01:09.115 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-27T02:01:09.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-27T02:01:09.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[21867]: osdmap e23: 4 total, 3 up, 4 in 2024-04-27T02:01:09.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:09.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[21867]: from='osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:01:09.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:01:09.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[26568]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-27T02:01:09.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[26568]: osdmap e23: 4 total, 3 up, 4 in 2024-04-27T02:01:09.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:09.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[26568]: from='osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:01:09.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-mon[26568]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:01:09.960 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[38337]: 2024-04-27T02:01:09.717+0000 7f3a24107700 -1 osd.3 0 waiting for initial osdmap 2024-04-27T02:01:09.960 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:01:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[38337]: 2024-04-27T02:01:09.721+0000 7f3a1ea9e700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:01:10.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:09 smithi151 ceph-mon[22663]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-27T02:01:10.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:09 smithi151 ceph-mon[22663]: osdmap e23: 4 total, 3 up, 4 in 2024-04-27T02:01:10.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:09 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:10.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:09 smithi151 ceph-mon[22663]: from='osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:01:10.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:09 smithi151 ceph-mon[22663]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:01:10.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:10 smithi151 ceph-mon[22663]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:01:10.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:10 smithi151 ceph-mon[22663]: osdmap e24: 4 total, 3 up, 4 in 2024-04-27T02:01:10.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:10 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:10.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:10 smithi151 ceph-mon[22663]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:01:10.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:10 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:10.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:01:10.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[21867]: osdmap e24: 4 total, 3 up, 4 in 2024-04-27T02:01:10.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:10.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[21867]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:01:10.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:10.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[26568]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-27T02:01:10.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[26568]: osdmap e24: 4 total, 3 up, 4 in 2024-04-27T02:01:10.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:10.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[26568]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:01:10.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:10 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:10.931 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-27T02:01:10.932 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-27T02:01:10.932 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-27T02:01:10.932 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-27T02:01:10.932 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124109 s, 845 MB/s 2024-04-27T02:01:10.932 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-27T02:01:11.449 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi151:vg_nvme/lv_4 2024-04-27T02:01:11.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[21867]: osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826] boot 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[21867]: osdmap e25: 4 total, 4 up, 4 in 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[26568]: osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826] boot 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[26568]: osdmap e25: 4 total, 4 up, 4 in 2024-04-27T02:01:11.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:11 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:11 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:01:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:11 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:01:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:11 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:11 smithi151 ceph-mon[22663]: osd.3 [v2:172.21.15.26:6826/1737241826,v1:172.21.15.26:6827/1737241826] boot 2024-04-27T02:01:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:11 smithi151 ceph-mon[22663]: osdmap e25: 4 total, 4 up, 4 in 2024-04-27T02:01:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:11 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:01:12.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:01:12.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: osdmap e26: 4 total, 4 up, 4 in 2024-04-27T02:01:12.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi026 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi026 to 4017M 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:12.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:12 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:13.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: osdmap e26: 4 total, 4 up, 4 in 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi026 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi026 to 4017M 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:13.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: osdmap e26: 4 total, 4 up, 4 in 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi026 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi026 to 4017M 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:13.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:12 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:13.987 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:13 smithi151 ceph-mon[22663]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:01:13.987 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:13 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:13.987 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:13 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:13.987 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:13 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[21867]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[26568]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:14.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:13 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:14.882 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:14 smithi151 ceph-mon[22663]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:15.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:14 smithi026 ceph-mon[21867]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:15.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:14 smithi026 ceph-mon[26568]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:16.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:15 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/3698402503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]: dispatch 2024-04-27T02:01:16.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:15 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]: dispatch 2024-04-27T02:01:16.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:15 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]': finished 2024-04-27T02:01:16.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:15 smithi151 ceph-mon[22663]: osdmap e27: 5 total, 4 up, 5 in 2024-04-27T02:01:16.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:15 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/3698402503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]: dispatch 2024-04-27T02:01:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]: dispatch 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]': finished 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[21867]: osdmap e27: 5 total, 4 up, 5 in 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/3698402503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]: dispatch 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]: dispatch 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "92076b2a-a5ba-4978-ab85-6b87fa05723f"}]': finished 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[26568]: osdmap e27: 5 total, 4 up, 5 in 2024-04-27T02:01:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:15 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:16 smithi151 ceph-mon[22663]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:16 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/2890653007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:01:17.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:16 smithi026 ceph-mon[26568]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:17.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:16 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/2890653007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:01:17.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:16 smithi026 ceph-mon[21867]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:17.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:16 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/2890653007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:01:19.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:19 smithi151 ceph-mon[22663]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:19.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:19 smithi026 ceph-mon[21867]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:19.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:19 smithi026 ceph-mon[26568]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:21.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:21 smithi151 ceph-mon[22663]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:21.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:21 smithi026 ceph-mon[21867]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:21.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:21 smithi026 ceph-mon[26568]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:22.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:22 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:01:22.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:22 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:22.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:22 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:01:22.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:22 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:22.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:22 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:01:22.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:22 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:23.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:23 smithi151 ceph-mon[22663]: Deploying daemon osd.4 on smithi151 2024-04-27T02:01:23.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:23 smithi151 ceph-mon[22663]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:23.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:23 smithi026 ceph-mon[21867]: Deploying daemon osd.4 on smithi151 2024-04-27T02:01:23.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:23 smithi026 ceph-mon[21867]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:23.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:23 smithi026 ceph-mon[26568]: Deploying daemon osd.4 on smithi151 2024-04-27T02:01:23.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:23 smithi026 ceph-mon[26568]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:25.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:25 smithi151 ceph-mon[22663]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:25.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:25 smithi026 ceph-mon[21867]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:25.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:25 smithi026 ceph-mon[26568]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[21867]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[26568]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:27.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:27 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:27.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:27 smithi151 ceph-mon[22663]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:27.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:27 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:27.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:27 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:27.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:27 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:27.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:27 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:29.222 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 4 on host 'smithi151' 2024-04-27T02:01:29.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:29 smithi151 ceph-mon[22663]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:29.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:29 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:29.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:29 smithi026 ceph-mon[21867]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:29.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:29 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:29.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:29 smithi026 ceph-mon[26568]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:29.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:29 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:29.889 DEBUG:teuthology.orchestra.run.smithi151:osd.4> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.4.service 2024-04-27T02:01:29.892 INFO:tasks.cephadm:Deploying osd.5 on smithi151 with /dev/vg_nvme/lv_3... 2024-04-27T02:01:29.892 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-27T02:01:30.567 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:30 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:30.567 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:30 smithi151 ceph-mon[22663]: from='osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:01:30.567 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:30 smithi151 ceph-mon[22663]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:01:30.567 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:30 smithi151 ceph-mon[22663]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:30.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:30.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[21867]: from='osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:01:30.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:01:30.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[21867]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:30.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:30.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[26568]: from='osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:01:30.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[26568]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:01:30.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:30 smithi026 ceph-mon[26568]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:31.497 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:31 smithi151 ceph-mon[22663]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-27T02:01:31.497 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:31 smithi151 ceph-mon[22663]: osdmap e28: 5 total, 4 up, 5 in 2024-04-27T02:01:31.497 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:31 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:31.497 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:31 smithi151 ceph-mon[22663]: from='osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:31.497 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:31 smithi151 ceph-mon[22663]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:31.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[26568]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[26568]: osdmap e28: 5 total, 4 up, 5 in 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[26568]: from='osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[26568]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[21867]: osdmap e28: 5 total, 4 up, 5 in 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[21867]: from='osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:31.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:31 smithi026 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:31.762 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:01:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[26043]: 2024-04-27T02:01:31.515+0000 7f24d8cf9700 -1 osd.4 0 waiting for initial osdmap 2024-04-27T02:01:31.762 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:01:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[26043]: 2024-04-27T02:01:31.520+0000 7f24d4e93700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:01:32.072 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-27T02:01:32.072 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-27T02:01:32.072 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-27T02:01:32.072 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-27T02:01:32.072 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0150072 s, 699 MB/s 2024-04-27T02:01:32.072 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-27T02:01:32.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:01:32.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: osdmap e29: 5 total, 4 up, 5 in 2024-04-27T02:01:32.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:32.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093] boot 2024-04-27T02:01:32.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: osdmap e30: 5 total, 5 up, 5 in 2024-04-27T02:01:32.645 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:32 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.668 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi151:vg_nvme/lv_3 2024-04-27T02:01:32.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: osdmap e29: 5 total, 4 up, 5 in 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093] boot 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: osdmap e30: 5 total, 5 up, 5 in 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: osdmap e29: 5 total, 4 up, 5 in 2024-04-27T02:01:32.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-27T02:01:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: osd.4 [v2:172.21.15.151:6800/1744678093,v1:172.21.15.151:6801/1744678093] boot 2024-04-27T02:01:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: osdmap e30: 5 total, 5 up, 5 in 2024-04-27T02:01:32.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:32 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:01:33.772 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:33 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:01:33.772 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:33 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:01:33.772 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:33 smithi151 ceph-mon[22663]: osdmap e31: 5 total, 5 up, 5 in 2024-04-27T02:01:33.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:33 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:01:33.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:33 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:01:33.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:33 smithi026 ceph-mon[21867]: osdmap e31: 5 total, 5 up, 5 in 2024-04-27T02:01:33.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:33 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:01:33.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:33 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:01:33.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:33 smithi026 ceph-mon[26568]: osdmap e31: 5 total, 5 up, 5 in 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: pgmap v107: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: osdmap e32: 5 total, 5 up, 5 in 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:35.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:35.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:35 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: pgmap v107: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: osdmap e32: 5 total, 5 up, 5 in 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: pgmap v107: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: osdmap e32: 5 total, 5 up, 5 in 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:35.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:35 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:36.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:36 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi151 2024-04-27T02:01:36.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:36 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi151 to 17088M 2024-04-27T02:01:36.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:36 smithi151 ceph-mon[22663]: osdmap e33: 5 total, 5 up, 5 in 2024-04-27T02:01:36.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:36 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:36.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:36 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:36.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:36 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi151 2024-04-27T02:01:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi151 to 17088M 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[21867]: osdmap e33: 5 total, 5 up, 5 in 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi151 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi151 to 17088M 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[26568]: osdmap e33: 5 total, 5 up, 5 in 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:36 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:37 smithi026 ceph-mon[21867]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:01:37.644 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:37 smithi026 ceph-mon[21867]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:37.645 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:37 smithi026 ceph-mon[21867]: osdmap e34: 5 total, 5 up, 5 in 2024-04-27T02:01:37.645 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:37 smithi026 ceph-mon[26568]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:01:37.645 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:37 smithi026 ceph-mon[26568]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:37.645 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:37 smithi026 ceph-mon[26568]: osdmap e34: 5 total, 5 up, 5 in 2024-04-27T02:01:37.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:37 smithi151 ceph-mon[22663]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:01:37.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:37 smithi151 ceph-mon[22663]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:37.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:37 smithi151 ceph-mon[22663]: osdmap e34: 5 total, 5 up, 5 in 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/766207588' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]: dispatch 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]: dispatch 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]': finished 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[21867]: osdmap e35: 6 total, 5 up, 6 in 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/766207588' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]: dispatch 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]: dispatch 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]': finished 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[26568]: osdmap e35: 6 total, 5 up, 6 in 2024-04-27T02:01:38.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:38 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:38.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:38 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/766207588' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]: dispatch 2024-04-27T02:01:38.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:38 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]: dispatch 2024-04-27T02:01:38.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:38 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93422b66-336c-490e-aa55-22c06d91a888"}]': finished 2024-04-27T02:01:38.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:38 smithi151 ceph-mon[22663]: osdmap e35: 6 total, 5 up, 6 in 2024-04-27T02:01:38.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:38 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:39 smithi026 ceph-mon[21867]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:39 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/1345172905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:01:39.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:39 smithi026 ceph-mon[26568]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:39.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:39 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/1345172905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:01:39.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:39 smithi151 ceph-mon[22663]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:39.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:39 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/1345172905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:01:40.629 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:40 smithi151 ceph-mon[22663]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-27T02:01:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:40 smithi026 ceph-mon[21867]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-27T02:01:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:40 smithi026 ceph-mon[26568]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-27T02:01:41.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:41 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:01:41.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:41 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:01:42.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:41 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:01:42.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:41 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:01:42.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:41 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:01:42.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:41 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:01:43.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:43 smithi151 ceph-mon[22663]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-27T02:01:43.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:43 smithi026 ceph-mon[21867]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-27T02:01:43.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:43 smithi026 ceph-mon[26568]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-27T02:01:44.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:01:44.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:44.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:01:44.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:44.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:01:44.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:45.573 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:45 smithi151 ceph-mon[22663]: Deploying daemon osd.5 on smithi151 2024-04-27T02:01:45.573 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:45 smithi151 ceph-mon[22663]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-27T02:01:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:45 smithi026 ceph-mon[21867]: Deploying daemon osd.5 on smithi151 2024-04-27T02:01:45.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:45 smithi026 ceph-mon[21867]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-27T02:01:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:45 smithi026 ceph-mon[26568]: Deploying daemon osd.5 on smithi151 2024-04-27T02:01:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:45 smithi026 ceph-mon[26568]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-27T02:01:47.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:47 smithi151 ceph-mon[22663]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-27T02:01:47.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:47 smithi026 ceph-mon[21867]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-27T02:01:47.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:47 smithi026 ceph-mon[26568]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-27T02:01:49.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:49 smithi151 ceph-mon[22663]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:01:49.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:49 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:49.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:49 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:49.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:49 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:49.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:49 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:49.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[21867]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:01:49.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:49.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:49.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:49.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:49.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[26568]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:01:49.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:49.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:49.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:49.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:49 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:51.325 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 5 on host 'smithi151' 2024-04-27T02:01:51.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:51 smithi151 ceph-mon[22663]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-27T02:01:51.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:51 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:51.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:51 smithi026 ceph-mon[26568]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-27T02:01:51.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:51 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:51.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:51 smithi026 ceph-mon[21867]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-27T02:01:51.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:51 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:51.920 DEBUG:teuthology.orchestra.run.smithi151:osd.5> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.5.service 2024-04-27T02:01:51.923 INFO:tasks.cephadm:Deploying osd.6 on smithi151 with /dev/vg_nvme/lv_2... 2024-04-27T02:01:51.923 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-27T02:01:52.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:52 smithi026 ceph-mon[21867]: from='osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:01:52.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:52 smithi026 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:01:52.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:52 smithi026 ceph-mon[26568]: from='osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:01:52.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:52 smithi026 ceph-mon[26568]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:01:52.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:52 smithi151 ceph-mon[22663]: from='osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:01:52.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:52 smithi151 ceph-mon[22663]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:01:53.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[21867]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[21867]: osdmap e36: 6 total, 5 up, 6 in 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[21867]: from='osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[26568]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[26568]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[26568]: osdmap e36: 6 total, 5 up, 6 in 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[26568]: from='osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:53.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:53 smithi026 ceph-mon[26568]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:53.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-mon[22663]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:53.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-mon[22663]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-27T02:01:53.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-mon[22663]: osdmap e36: 6 total, 5 up, 6 in 2024-04-27T02:01:53.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:53.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-mon[22663]: from='osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:53.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-mon[22663]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:01:53.668 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[28995]: 2024-04-27T02:01:53.385+0000 7f9225a55700 -1 osd.5 0 waiting for initial osdmap 2024-04-27T02:01:53.668 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:01:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[28995]: 2024-04-27T02:01:53.390+0000 7f92203ec700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:01:54.289 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-27T02:01:54.290 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-27T02:01:54.290 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-27T02:01:54.290 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-27T02:01:54.290 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0184703 s, 568 MB/s 2024-04-27T02:01:54.290 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-27T02:01:54.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:54 smithi151 ceph-mon[22663]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:01:54.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:54 smithi151 ceph-mon[22663]: osdmap e37: 6 total, 5 up, 6 in 2024-04-27T02:01:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:54 smithi151 ceph-mon[22663]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:54.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[21867]: osdmap e37: 6 total, 5 up, 6 in 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[21867]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[26568]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[26568]: osdmap e37: 6 total, 5 up, 6 in 2024-04-27T02:01:54.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:54.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:54.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[26568]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-27T02:01:54.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:55.506 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:55 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:01:55.507 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:55 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:01:55.507 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:55 smithi151 ceph-mon[22663]: osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978] boot 2024-04-27T02:01:55.507 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:55 smithi151 ceph-mon[22663]: osdmap e38: 6 total, 6 up, 6 in 2024-04-27T02:01:55.507 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:55 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:55.533 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi151:vg_nvme/lv_2 2024-04-27T02:01:55.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[21867]: osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978] boot 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[21867]: osdmap e38: 6 total, 6 up, 6 in 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[26568]: osd.5 [v2:172.21.15.151:6808/3769734978,v1:172.21.15.151:6809/3769734978] boot 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[26568]: osdmap e38: 6 total, 6 up, 6 in 2024-04-27T02:01:55.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:55 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:01:56.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:56 smithi151 ceph-mon[22663]: osdmap e39: 6 total, 6 up, 6 in 2024-04-27T02:01:56.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:56 smithi151 ceph-mon[22663]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:01:56.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:56 smithi026 ceph-mon[21867]: osdmap e39: 6 total, 6 up, 6 in 2024-04-27T02:01:56.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:56 smithi026 ceph-mon[21867]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:01:56.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:56 smithi026 ceph-mon[26568]: osdmap e39: 6 total, 6 up, 6 in 2024-04-27T02:01:56.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:56 smithi026 ceph-mon[26568]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:01:57.688 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:57 smithi151 ceph-mon[22663]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-27T02:01:57.688 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:57 smithi151 ceph-mon[22663]: osdmap e40: 6 total, 6 up, 6 in 2024-04-27T02:01:57.688 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:57 smithi151 ceph-mon[22663]: osdmap e41: 6 total, 6 up, 6 in 2024-04-27T02:01:57.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:57 smithi026 ceph-mon[21867]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-27T02:01:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:57 smithi026 ceph-mon[21867]: osdmap e40: 6 total, 6 up, 6 in 2024-04-27T02:01:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:57 smithi026 ceph-mon[21867]: osdmap e41: 6 total, 6 up, 6 in 2024-04-27T02:01:57.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:57 smithi026 ceph-mon[26568]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-27T02:01:57.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:57 smithi026 ceph-mon[26568]: osdmap e40: 6 total, 6 up, 6 in 2024-04-27T02:01:57.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:57 smithi026 ceph-mon[26568]: osdmap e41: 6 total, 6 up, 6 in 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: osdmap e42: 6 total, 6 up, 6 in 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: osdmap e42: 6 total, 6 up, 6 in 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:59.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:59.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:01:59 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: osdmap e42: 6 total, 6 up, 6 in 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:01:59.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:01:59.831 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:01:59.831 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:01:59 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:00.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:00 smithi026 ceph-mon[21867]: from='client.24260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:00.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:00 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi151 2024-04-27T02:02:00.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:00 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi151 to 8544M 2024-04-27T02:02:00.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:00 smithi026 ceph-mon[26568]: from='client.24260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:00.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:00 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi151 2024-04-27T02:02:00.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:00 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi151 to 8544M 2024-04-27T02:02:00.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:00 smithi151 ceph-mon[22663]: from='client.24260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:00.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:00 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi151 2024-04-27T02:02:00.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:00 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi151 to 8544M 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[21867]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/1726726140' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]: dispatch 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]: dispatch 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]': finished 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[21867]: osdmap e43: 7 total, 6 up, 7 in 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[26568]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/1726726140' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]: dispatch 2024-04-27T02:02:01.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]: dispatch 2024-04-27T02:02:01.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]': finished 2024-04-27T02:02:01.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[26568]: osdmap e43: 7 total, 6 up, 7 in 2024-04-27T02:02:01.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:01 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:01.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:01 smithi151 ceph-mon[22663]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:01.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:01 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/1726726140' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]: dispatch 2024-04-27T02:02:01.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:01 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]: dispatch 2024-04-27T02:02:01.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:01 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ffe8da7-1406-4415-a422-bd87364d204a"}]': finished 2024-04-27T02:02:01.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:01 smithi151 ceph-mon[22663]: osdmap e43: 7 total, 6 up, 7 in 2024-04-27T02:02:01.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:01 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:02.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:02 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/3130354916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:02:02.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:02 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/3130354916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:02:02.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:02 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/3130354916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:02:03.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:03 smithi151 ceph-mon[22663]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-27T02:02:03.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:03 smithi151 ceph-mon[22663]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-27T02:02:03.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:03 smithi151 ceph-mon[22663]: Cluster is now healthy 2024-04-27T02:02:03.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:03 smithi026 ceph-mon[21867]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-27T02:02:03.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:03 smithi026 ceph-mon[21867]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-27T02:02:03.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:03 smithi026 ceph-mon[21867]: Cluster is now healthy 2024-04-27T02:02:03.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:03 smithi026 ceph-mon[26568]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-27T02:02:03.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:03 smithi026 ceph-mon[26568]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-27T02:02:03.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:03 smithi026 ceph-mon[26568]: Cluster is now healthy 2024-04-27T02:02:04.793 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:04 smithi151 ceph-mon[22663]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-27T02:02:04.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:04 smithi026 ceph-mon[21867]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-27T02:02:04.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:04 smithi026 ceph-mon[26568]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-27T02:02:07.058 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:06 smithi151 ceph-mon[22663]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-27T02:02:07.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:06 smithi026 ceph-mon[21867]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-27T02:02:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:06 smithi026 ceph-mon[26568]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-27T02:02:08.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:07 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:02:08.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:07 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:08.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:07 smithi151 ceph-mon[22663]: Deploying daemon osd.6 on smithi151 2024-04-27T02:02:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:07 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:02:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:07 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:07 smithi026 ceph-mon[21867]: Deploying daemon osd.6 on smithi151 2024-04-27T02:02:08.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:07 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:02:08.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:07 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:08.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:07 smithi026 ceph-mon[26568]: Deploying daemon osd.6 on smithi151 2024-04-27T02:02:09.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:09 smithi151 ceph-mon[22663]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-27T02:02:09.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:09 smithi026 ceph-mon[21867]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-27T02:02:09.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:09 smithi026 ceph-mon[26568]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-27T02:02:11.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:11 smithi151 ceph-mon[22663]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:02:11.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:11 smithi026 ceph-mon[21867]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:02:11.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:11 smithi026 ceph-mon[26568]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:02:13.324 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:13 smithi151 ceph-mon[22663]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:13.324 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:13 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:13.324 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:13 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:13.324 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:13 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:13.324 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:13 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[21867]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[26568]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:13 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:15.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:15 smithi151 ceph-mon[22663]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:15.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:15 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:15.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:15 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:15 smithi026 ceph-mon[21867]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:15.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:15 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:15.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:15 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:15.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:15 smithi026 ceph-mon[26568]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:15.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:15 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:15.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:15 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:15.491 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 6 on host 'smithi151' 2024-04-27T02:02:16.161 DEBUG:teuthology.orchestra.run.smithi151:osd.6> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.6.service 2024-04-27T02:02:16.164 INFO:tasks.cephadm:Deploying osd.7 on smithi151 with /dev/vg_nvme/lv_1... 2024-04-27T02:02:16.164 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-27T02:02:16.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:16 smithi151 ceph-mon[22663]: from='osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:02:16.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:16 smithi151 ceph-mon[22663]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:02:16.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:16 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:16.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:16 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:16.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:16 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:16.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:16 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:16.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[21867]: from='osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:02:16.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:02:16.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:16.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:16.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:16.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:16.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[26568]: from='osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:02:16.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[26568]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:02:16.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:16.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:16.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:16.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:16 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:17.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-mon[22663]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:17.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-mon[22663]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-27T02:02:17.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-mon[22663]: osdmap e44: 7 total, 6 up, 7 in 2024-04-27T02:02:17.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:17.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-mon[22663]: from='osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:17.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-mon[22663]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:17.283 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[31876]: 2024-04-27T02:02:17.091+0000 7f92d4705700 -1 osd.6 0 waiting for initial osdmap 2024-04-27T02:02:17.283 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:02:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[31876]: 2024-04-27T02:02:17.097+0000 7f92cd098700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:02:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[21867]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[21867]: osdmap e44: 7 total, 6 up, 7 in 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[21867]: from='osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[26568]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[26568]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[26568]: osdmap e44: 7 total, 6 up, 7 in 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[26568]: from='osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:17.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:17 smithi026 ceph-mon[26568]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:18.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:18 smithi151 ceph-mon[22663]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:02:18.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:18 smithi151 ceph-mon[22663]: osdmap e45: 7 total, 6 up, 7 in 2024-04-27T02:02:18.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:18 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:18 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:18 smithi151 ceph-mon[22663]: osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847] boot 2024-04-27T02:02:18.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:18 smithi151 ceph-mon[22663]: osdmap e46: 7 total, 7 up, 7 in 2024-04-27T02:02:18.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:18 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:02:18.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[21867]: osdmap e45: 7 total, 6 up, 7 in 2024-04-27T02:02:18.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[21867]: osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847] boot 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[21867]: osdmap e46: 7 total, 7 up, 7 in 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[26568]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[26568]: osdmap e45: 7 total, 6 up, 7 in 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[26568]: osd.6 [v2:172.21.15.151:6816/2406497847,v1:172.21.15.151:6817/2406497847] boot 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[26568]: osdmap e46: 7 total, 7 up, 7 in 2024-04-27T02:02:18.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:18 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:02:19.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:19 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:02:19.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:19 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:02:19.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:19 smithi151 ceph-mon[22663]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:19.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:19 smithi151 ceph-mon[22663]: osdmap e47: 7 total, 7 up, 7 in 2024-04-27T02:02:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:02:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:02:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[21867]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[21867]: osdmap e47: 7 total, 7 up, 7 in 2024-04-27T02:02:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:02:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:02:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[26568]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-27T02:02:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:19 smithi026 ceph-mon[26568]: osdmap e47: 7 total, 7 up, 7 in 2024-04-27T02:02:19.489 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-27T02:02:19.489 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-27T02:02:19.490 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-27T02:02:19.490 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-27T02:02:19.490 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0134979 s, 777 MB/s 2024-04-27T02:02:19.490 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-27T02:02:20.252 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi151:vg_nvme/lv_1 2024-04-27T02:02:20.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:20 smithi151 ceph-mon[22663]: osdmap e48: 7 total, 7 up, 7 in 2024-04-27T02:02:20.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:20 smithi026 ceph-mon[21867]: osdmap e48: 7 total, 7 up, 7 in 2024-04-27T02:02:20.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:20 smithi026 ceph-mon[26568]: osdmap e48: 7 total, 7 up, 7 in 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: osdmap e49: 7 total, 7 up, 7 in 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi151 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi151 to 5696M 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:21 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: osdmap e49: 7 total, 7 up, 7 in 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi151 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi151 to 5696M 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: osdmap e49: 7 total, 7 up, 7 in 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi151 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi151 to 5696M 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:21 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:22.820 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:22 smithi151 ceph-mon[22663]: osdmap e50: 7 total, 7 up, 7 in 2024-04-27T02:02:22.821 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:22 smithi151 ceph-mon[22663]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:22.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:22 smithi026 ceph-mon[21867]: osdmap e50: 7 total, 7 up, 7 in 2024-04-27T02:02:22.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:22 smithi026 ceph-mon[21867]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:22.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:22 smithi026 ceph-mon[26568]: osdmap e50: 7 total, 7 up, 7 in 2024-04-27T02:02:22.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:22 smithi026 ceph-mon[26568]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:23.828 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:23 smithi151 ceph-mon[22663]: from='client.24287 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:23.828 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:23 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:02:23.828 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:23 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:02:23.828 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:23 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:23.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[21867]: from='client.24287 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:23.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:02:23.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:02:23.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:23.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[26568]: from='client.24287 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:23.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-27T02:02:23.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-27T02:02:23.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:23 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:24.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:24 smithi151 ceph-mon[22663]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-27T02:02:24.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:24 smithi026 ceph-mon[21867]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-27T02:02:24.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:24 smithi026 ceph-mon[26568]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-27T02:02:25.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:25 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/2174658685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]: dispatch 2024-04-27T02:02:25.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:25 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]: dispatch 2024-04-27T02:02:25.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:25 smithi151 ceph-mon[22663]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]': finished 2024-04-27T02:02:25.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:25 smithi151 ceph-mon[22663]: osdmap e51: 8 total, 7 up, 8 in 2024-04-27T02:02:25.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:25 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/2174658685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]: dispatch 2024-04-27T02:02:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]: dispatch 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]': finished 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[21867]: osdmap e51: 8 total, 7 up, 8 in 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/2174658685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]: dispatch 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]: dispatch 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[26568]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef6a01df-7c83-4775-a7c7-7f85b885ac12"}]': finished 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[26568]: osdmap e51: 8 total, 7 up, 8 in 2024-04-27T02:02:25.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:25 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:26.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:26 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/1803976659' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:02:26.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:26 smithi151 ceph-mon[22663]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-27T02:02:26.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:26 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/1803976659' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:02:26.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:26 smithi026 ceph-mon[21867]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-27T02:02:26.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:26 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/1803976659' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-27T02:02:26.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:26 smithi026 ceph-mon[26568]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-27T02:02:29.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:29 smithi151 ceph-mon[22663]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-27T02:02:29.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:29 smithi026 ceph-mon[21867]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-27T02:02:29.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:29 smithi026 ceph-mon[26568]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-27T02:02:31.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:31 smithi026 ceph-mon[21867]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-27T02:02:31.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:31 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:02:31.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:31 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:31.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:31 smithi026 ceph-mon[26568]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-27T02:02:31.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:31 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:02:31.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:31 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:31.630 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:31 smithi151 ceph-mon[22663]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-27T02:02:31.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:31 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:02:31.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:31 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:32.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:32 smithi026 ceph-mon[21867]: Deploying daemon osd.7 on smithi151 2024-04-27T02:02:32.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:32 smithi026 ceph-mon[26568]: Deploying daemon osd.7 on smithi151 2024-04-27T02:02:32.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:32 smithi151 ceph-mon[22663]: Deploying daemon osd.7 on smithi151 2024-04-27T02:02:33.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:33 smithi026 ceph-mon[21867]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:02:33.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:33 smithi026 ceph-mon[26568]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:02:33.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:33 smithi151 ceph-mon[22663]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-27T02:02:35.709 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:35 smithi151 ceph-mon[22663]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:35.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:35 smithi026 ceph-mon[21867]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:35.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:35 smithi026 ceph-mon[26568]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:37.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:37 smithi151 ceph-mon[22663]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:37.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:37 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[21867]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[26568]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:37 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:39.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:39 smithi151 ceph-mon[22663]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:39.475 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 7 on host 'smithi151' 2024-04-27T02:02:39.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:39 smithi026 ceph-mon[21867]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:39.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:39 smithi026 ceph-mon[26568]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:40.068 DEBUG:teuthology.orchestra.run.smithi151:osd.7> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.7.service 2024-04-27T02:02:40.071 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-27T02:02:40.071 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-27T02:02:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:40 smithi151 ceph-mon[22663]: from='osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:02:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:40 smithi151 ceph-mon[22663]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:02:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:40 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:40.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[26568]: from='osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:02:40.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[26568]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:02:40.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:40.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:40.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[21867]: from='osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:02:40.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:02:40.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:40.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:40 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:41.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[21867]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:41.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-27T02:02:41.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[21867]: osdmap e52: 8 total, 7 up, 8 in 2024-04-27T02:02:41.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:41.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[21867]: from='osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:41.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:41.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[26568]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:41.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[26568]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-27T02:02:41.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[26568]: osdmap e52: 8 total, 7 up, 8 in 2024-04-27T02:02:41.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:41.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[26568]: from='osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:41.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:41 smithi026 ceph-mon[26568]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:41.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-mon[22663]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:41.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-mon[22663]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-27T02:02:41.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-mon[22663]: osdmap e52: 8 total, 7 up, 8 in 2024-04-27T02:02:41.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:41.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-mon[22663]: from='osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:41.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-mon[22663]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:02:41.600 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[34804]: 2024-04-27T02:02:41.387+0000 7ff10c9c4700 -1 osd.7 0 waiting for initial osdmap 2024-04-27T02:02:41.600 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:02:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[34804]: 2024-04-27T02:02:41.394+0000 7ff106b5a700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:02:42.232 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: purged_snaps scrub starts 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: purged_snaps scrub ok 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: osdmap e53: 8 total, 7 up, 8 in 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:02:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:02:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3000451906' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-27T02:02:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593] boot 2024-04-27T02:02:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: osdmap e54: 8 total, 8 up, 8 in 2024-04-27T02:02:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:42 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: purged_snaps scrub starts 2024-04-27T02:02:42.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: purged_snaps scrub ok 2024-04-27T02:02:42.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: osdmap e53: 8 total, 7 up, 8 in 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3000451906' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593] boot 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: osdmap e54: 8 total, 8 up, 8 in 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.690 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: purged_snaps scrub starts 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: purged_snaps scrub ok 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: osdmap e53: 8 total, 7 up, 8 in 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3000451906' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: osd.7 [v2:172.21.15.151:6824/3440470593,v1:172.21.15.151:6825/3440470593] boot 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: osdmap e54: 8 total, 8 up, 8 in 2024-04-27T02:02:42.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:42 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:02:42.972 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1714183337,"num_in_osds":8,"osd_in_since":1714183344,"num_remapped_pgs":0} 2024-04-27T02:02:43.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:43 smithi151 ceph-mon[22663]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:43.601 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:43 smithi151 ceph-mon[22663]: osdmap e55: 8 total, 8 up, 8 in 2024-04-27T02:02:43.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:43 smithi026 ceph-mon[21867]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:43.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:43 smithi026 ceph-mon[21867]: osdmap e55: 8 total, 8 up, 8 in 2024-04-27T02:02:43.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:43 smithi026 ceph-mon[26568]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-27T02:02:43.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:43 smithi026 ceph-mon[26568]: osdmap e55: 8 total, 8 up, 8 in 2024-04-27T02:02:43.974 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi151 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi151 to 4272M 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:44.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[21867]: osdmap e56: 8 total, 8 up, 8 in 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi151 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi151 to 4272M 2024-04-27T02:02:44.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:44.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:44.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:44.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:44.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:02:44.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:44 smithi026 ceph-mon[26568]: osdmap e56: 8 total, 8 up, 8 in 2024-04-27T02:02:45.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: Detected new or changed devices on smithi151 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi151 to 4272M 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:45.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:02:45.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:44 smithi151 ceph-mon[22663]: osdmap e56: 8 total, 8 up, 8 in 2024-04-27T02:02:46.258 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:02:46.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:46 smithi026 ceph-mon[21867]: osdmap e57: 8 total, 8 up, 8 in 2024-04-27T02:02:46.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:46 smithi026 ceph-mon[21867]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:02:46.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:46 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1678833707' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-27T02:02:46.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:46 smithi026 ceph-mon[26568]: osdmap e57: 8 total, 8 up, 8 in 2024-04-27T02:02:46.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:46 smithi026 ceph-mon[26568]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:02:46.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:46 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1678833707' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-27T02:02:46.847 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1714183362,"num_in_osds":8,"osd_in_since":1714183344,"num_remapped_pgs":0} 2024-04-27T02:02:46.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-27T02:02:47.067 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:02:47.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:46 smithi151 ceph-mon[22663]: osdmap e57: 8 total, 8 up, 8 in 2024-04-27T02:02:47.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:46 smithi151 ceph-mon[22663]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:02:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:46 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1678833707' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-27T02:02:47.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:47 smithi026 ceph-mon[21867]: osdmap e58: 8 total, 8 up, 8 in 2024-04-27T02:02:47.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:47 smithi026 ceph-mon[26568]: osdmap e58: 8 total, 8 up, 8 in 2024-04-27T02:02:48.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:47 smithi151 ceph-mon[22663]: osdmap e58: 8 total, 8 up, 8 in 2024-04-27T02:02:48.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:48 smithi026 ceph-mon[21867]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-27T02:02:48.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:48 smithi026 ceph-mon[26568]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-27T02:02:48.877 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:02:48.877 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":58,"fsid":"7a5462be-0439-11ef-bc93-c7b262605968","created":"2024-04-27T01:57:47.310360+0000","modified":"2024-04-27T02:02:46.664820+0000","last_up_change":"2024-04-27T02:02:42.369042+0000","last_in_change":"2024-04-27T02:02:24.790754+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-27T02:00:46.196957+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5b7a48d3-2b4e-4b09-a4e7-819387c92ab3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6803","nonce":652787281}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6805","nonce":652787281}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6809","nonce":652787281}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6807","nonce":652787281}]},"public_addr":"172.21.15.26:6803/652787281","cluster_addr":"172.21.15.26:6805/652787281","heartbeat_back_addr":"172.21.15.26:6809/652787281","heartbeat_front_addr":"172.21.15.26:6807/652787281","state":["exists","up"]},{"osd":1,"uuid":"c9ce276b-a815-42f7-817b-2af147f9f597","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6811","nonce":1040402840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6813","nonce":1040402840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6817","nonce":1040402840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6815","nonce":1040402840}]},"public_addr":"172.21.15.26:6811/1040402840","cluster_addr":"172.21.15.26:6813/1040402840","heartbeat_back_addr":"172.21.15.26:6817/1040402840","heartbeat_front_addr":"172.21.15.26:6815/1040402840","state":["exists","up"]},{"osd":2,"uuid":"e22a75e4-af45-43bb-bf21-ab69c2046a4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6819","nonce":1403075570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6821","nonce":1403075570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6825","nonce":1403075570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6823","nonce":1403075570}]},"public_addr":"172.21.15.26:6819/1403075570","cluster_addr":"172.21.15.26:6821/1403075570","heartbeat_back_addr":"172.21.15.26:6825/1403075570","heartbeat_front_addr":"172.21.15.26:6823/1403075570","state":["exists","up"]},{"osd":3,"uuid":"a048a9f8-c1a3-4da7-908c-175069e4e4cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6827","nonce":1737241826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6829","nonce":1737241826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6833","nonce":1737241826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6831","nonce":1737241826}]},"public_addr":"172.21.15.26:6827/1737241826","cluster_addr":"172.21.15.26:6829/1737241826","heartbeat_back_addr":"172.21.15.26:6833/1737241826","heartbeat_front_addr":"172.21.15.26:6831/1737241826","state":["exists","up"]},{"osd":4,"uuid":"92076b2a-a5ba-4978-ab85-6b87fa05723f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6801","nonce":1744678093}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6803","nonce":1744678093}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6807","nonce":1744678093}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6805","nonce":1744678093}]},"public_addr":"172.21.15.151:6801/1744678093","cluster_addr":"172.21.15.151:6803/1744678093","heartbeat_back_addr":"172.21.15.151:6807/1744678093","heartbeat_front_addr":"172.21.15.151:6805/1744678093","state":["exists","up"]},{"osd":5,"uuid":"93422b66-336c-490e-aa55-22c06d91a888","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6809","nonce":3769734978}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6811","nonce":3769734978}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6815","nonce":3769734978}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6813","nonce":3769734978}]},"public_addr":"172.21.15.151:6809/3769734978","cluster_addr":"172.21.15.151:6811/3769734978","heartbeat_back_addr":"172.21.15.151:6815/3769734978","heartbeat_front_addr":"172.21.15.151:6813/3769734978","state":["exists","up"]},{"osd":6,"uuid":"2ffe8da7-1406-4415-a422-bd87364d204a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6817","nonce":2406497847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6819","nonce":2406497847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6823","nonce":2406497847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6821","nonce":2406497847}]},"public_addr":"172.21.15.151:6817/2406497847","cluster_addr":"172.21.15.151:6819/2406497847","heartbeat_back_addr":"172.21.15.151:6823/2406497847","heartbeat_front_addr":"172.21.15.151:6821/2406497847","state":["exists","up"]},{"osd":7,"uuid":"ef6a01df-7c83-4775-a7c7-7f85b885ac12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6825","nonce":3440470593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6827","nonce":3440470593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6831","nonce":3440470593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6829","nonce":3440470593}]},"public_addr":"172.21.15.151:6825/3440470593","cluster_addr":"172.21.15.151:6827/3440470593","heartbeat_back_addr":"172.21.15.151:6831/3440470593","heartbeat_front_addr":"172.21.15.151:6829/3440470593","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T01:59:57.301422+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:00:20.149531+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:00:43.693071+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:09.029421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:30.542566+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:52.602062+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:02:16.204412+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:02:40.427748+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/124039062":"2024-04-28T01:58:41.640278+0000","172.21.15.26:0/358798680":"2024-04-28T01:58:41.640278+0000","172.21.15.26:0/102542096":"2024-04-28T01:58:19.403412+0000","172.21.15.26:0/2505340537":"2024-04-28T01:58:19.403412+0000","172.21.15.26:6800/518236597":"2024-04-28T01:58:19.403412+0000","172.21.15.26:6801/1573366204":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6800/1573366204":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6801/518236597":"2024-04-28T01:58:19.403412+0000","172.21.15.26:0/4281274133":"2024-04-28T01:58:41.640278+0000","172.21.15.26:0/1738905563":"2024-04-28T01:58:19.403412+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-27T02:02:49.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:48 smithi151 ceph-mon[22663]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-27T02:02:49.401 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-27T02:00:46.196957+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-27T02:02:49.402 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-27T02:02:49.619 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:02:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:49 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1366713033' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:02:49.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:49 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1366713033' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:02:50.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:49 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1366713033' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:02:50.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:50 smithi026 ceph-mon[21867]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-27T02:02:50.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:50 smithi026 ceph-mon[26568]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-27T02:02:51.001 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:50 smithi151 ceph-mon[22663]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-27T02:02:51.470 INFO:teuthology.orchestra.run.smithi026.stdout:pg_num: 1 2024-04-27T02:02:51.783 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:51 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2972475276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-27T02:02:51.783 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:51 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2972475276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-27T02:02:52.100 INFO:tasks.cephadm:Adding prometheus.a on smithi151 2024-04-27T02:02:52.100 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi151=a' 2024-04-27T02:02:52.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:51 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2972475276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-27T02:02:53.506 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:53 smithi151 ceph-mon[22663]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-27T02:02:53.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:53 smithi026 ceph-mon[21867]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-27T02:02:53.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:53 smithi026 ceph-mon[26568]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-27T02:02:53.898 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled prometheus update... 2024-04-27T02:02:54.669 DEBUG:teuthology.orchestra.run.smithi151:prometheus.a> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-27T02:02:54.671 INFO:tasks.cephadm:Adding node-exporter.a on smithi026 2024-04-27T02:02:54.672 INFO:tasks.cephadm:Adding node-exporter.b on smithi151 2024-04-27T02:02:54.672 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi026=a;smithi151=b' 2024-04-27T02:02:55.175 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ignoring --setuser ceph since I am not root 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ignoring --setgroup ceph since I am not root 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mgr[23848]: -- 172.21.15.151:0/2370424802 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5586ecf06340 con 0x5586ecedec00 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:55.086+0000 7f154c87f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: Saving service prometheus spec with placement smithi151=a;count:1 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:55.176 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:54 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: Saving service prometheus spec with placement smithi151=a;count:1 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-27T02:02:55.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: Saving service prometheus spec with placement smithi151=a;count:1 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setuser ceph since I am not root 2024-04-27T02:02:55.218 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setgroup ceph since I am not root 2024-04-27T02:02:55.219 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:54 smithi026 ceph-mgr[22100]: -- 172.21.15.26:0/3094011225 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e63801c340 con 0x55e637ff4c00 2024-04-27T02:02:55.219 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:55.101+0000 7fcc96da1000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:02:55.219 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:55.235+0000 7fcc96da1000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:02:55.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:55.221+0000 7f154c87f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:02:55.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:55.320+0000 7f154c87f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:02:55.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:55.337+0000 7fcc96da1000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:02:55.907 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:55.656+0000 7fcc96da1000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:02:55.907 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:55.742+0000 7fcc96da1000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:02:55.908 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:55.634+0000 7f154c87f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:02:55.908 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:55.720+0000 7f154c87f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:02:56.197 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:55 smithi151 ceph-mon[22663]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-27T02:02:56.197 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:02:55 smithi151 ceph-mon[22663]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-27T02:02:56.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-mon[21867]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-27T02:02:56.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-mon[21867]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-27T02:02:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-mon[26568]: from='mgr.14150 172.21.15.26:0/2805691957' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-27T02:02:56.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:02:55 smithi026 ceph-mon[26568]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-27T02:02:57.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:56.868+0000 7f154c87f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:02:57.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:57.029+0000 7f154c87f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:02:57.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:57.109+0000 7f154c87f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:02:57.163 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:56.925+0000 7fcc96da1000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:02:57.163 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:57.096+0000 7fcc96da1000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:02:57.163 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:57.181+0000 7fcc96da1000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:02:57.818 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:57.419+0000 7f154c87f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:02:57.818 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:57.509+0000 7f154c87f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:02:57.818 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:57.599+0000 7f154c87f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:02:57.818 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:57.837+0000 7f154c87f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:02:57.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:57.510+0000 7fcc96da1000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:02:57.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:57.605+0000 7fcc96da1000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:02:57.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:57.702+0000 7fcc96da1000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:02:58.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:57.954+0000 7fcc96da1000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:02:58.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:58.274+0000 7f154c87f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:02:58.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:58.419+0000 7fcc96da1000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:02:59.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:58.752+0000 7f154c87f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:02:59.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:58.843+0000 7f154c87f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:02:59.314 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:58.927+0000 7fcc96da1000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:02:59.314 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:59.025+0000 7fcc96da1000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:02:59.314 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:59.333+0000 7fcc96da1000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:02:59.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:59.131+0000 7f154c87f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:02:59.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:59.219+0000 7f154c87f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:02:59.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:59.426+0000 7fcc96da1000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:02:59.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:59.617+0000 7f154c87f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:02:59.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:59.748+0000 7f154c87f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:02:59.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:59.838+0000 7f154c87f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:03:00.108 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:59.848+0000 7fcc96da1000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:03:00.108 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:02:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:02:59.987+0000 7fcc96da1000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:03:00.108 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:03:00.084+0000 7fcc96da1000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:03:00.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:02:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:02:59.949+0000 7f154c87f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:03:00.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:03:00.036+0000 7f154c87f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:03:00.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:03:00.204+0000 7fcc96da1000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:03:00.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:03:00.297+0000 7fcc96da1000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:03:00.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:03:00.573+0000 7f154c87f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:03:01.108 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:03:00.872+0000 7fcc96da1000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:03:01.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[26568]: Standby manager daemon x restarted 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[21867]: Standby manager daemon x started 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:03:01.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:00 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:03:01.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-mon[22663]: Standby manager daemon x restarted 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-mon[22663]: Standby manager daemon x started 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1991052014' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:03:00.930+0000 7f154c87f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:00 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: [27/Apr/2024:02:03:00] ENGINE Bus STARTING 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: CherryPy Checker: 2024-04-27T02:03:01.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: The Application mounted at '' has an empty config. 2024-04-27T02:03:01.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: 2024-04-27T02:03:01.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: [27/Apr/2024:02:03:01] ENGINE Serving on http://:::9283 2024-04-27T02:03:01.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: [27/Apr/2024:02:03:01] ENGINE Bus STARTED 2024-04-27T02:03:01.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:01 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:03:01.257+0000 7fcc96da1000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:03:02.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[26568]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[26568]: Active manager daemon y restarted 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[26568]: Activating manager daemon y 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[26568]: osdmap e59: 8 total, 8 up, 8 in 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[21867]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[21867]: Active manager daemon y restarted 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[21867]: Activating manager daemon y 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-mon[21867]: osdmap e59: 8 total, 8 up, 8 in 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:03:02] ENGINE Bus STARTING 2024-04-27T02:03:02.283 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: CherryPy Checker: 2024-04-27T02:03:02.284 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: The Application mounted at '' has an empty config. 2024-04-27T02:03:02.284 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:03:02.284 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:03:02] ENGINE Serving on http://:::9283 2024-04-27T02:03:02.284 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:03:02] ENGINE Bus STARTED 2024-04-27T02:03:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:02 smithi151 ceph-mon[22663]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-27T02:03:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:02 smithi151 ceph-mon[22663]: Active manager daemon y restarted 2024-04-27T02:03:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:02 smithi151 ceph-mon[22663]: Activating manager daemon y 2024-04-27T02:03:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:02 smithi151 ceph-mon[22663]: osdmap e59: 8 total, 8 up, 8 in 2024-04-27T02:03:03.002 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled node-exporter update... 2024-04-27T02:03:03.063 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:03:02] ENGINE Bus STARTING 2024-04-27T02:03:03.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: mgrmap e18: y(active, starting, since 0.73764s), standbys: x 2024-04-27T02:03:03.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:03:03.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:03:03.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:03:03.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: Manager daemon y is now available 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:03.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[21867]: [27/Apr/2024:02:03:02] ENGINE Bus STARTING 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: mgrmap e18: y(active, starting, since 0.73764s), standbys: x 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:03:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: Manager daemon y is now available 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:03:03.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:03:03.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:03.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:03:03.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:03:03.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-mon[26568]: [27/Apr/2024:02:03:02] ENGINE Bus STARTING 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: mgrmap e18: y(active, starting, since 0.73764s), standbys: x 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:03:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: Manager daemon y is now available 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:03:03.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:03 smithi151 ceph-mon[22663]: [27/Apr/2024:02:03:02] ENGINE Bus STARTING 2024-04-27T02:03:03.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:03:03] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:03:03.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:03:03] ENGINE Bus STARTED 2024-04-27T02:03:03.877 DEBUG:teuthology.orchestra.run.smithi026:node-exporter.a> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-27T02:03:03.880 DEBUG:teuthology.orchestra.run.smithi151:node-exporter.b> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-27T02:03:03.882 INFO:tasks.cephadm:Adding alertmanager.a on smithi026 2024-04-27T02:03:03.883 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi026=a' 2024-04-27T02:03:04.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: mgrmap e19: y(active, since 1.75282s), standbys: x 2024-04-27T02:03:04.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:04.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: Saving service node-exporter spec with placement smithi026=a;smithi151=b;count:2 2024-04-27T02:03:04.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:04.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: [27/Apr/2024:02:03:03] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: [27/Apr/2024:02:03:03] ENGINE Bus STARTED 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: mgrmap e19: y(active, since 1.75282s), standbys: x 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: Saving service node-exporter spec with placement smithi026=a;smithi151=b;count:2 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: [27/Apr/2024:02:03:03] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: [27/Apr/2024:02:03:03] ENGINE Bus STARTED 2024-04-27T02:03:04.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:04 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.125 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: mgrmap e19: y(active, since 1.75282s), standbys: x 2024-04-27T02:03:04.125 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:04.125 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: Saving service node-exporter spec with placement smithi026=a;smithi151=b;count:2 2024-04-27T02:03:04.125 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:04.125 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.126 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: [27/Apr/2024:02:03:03] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:03:04.126 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: [27/Apr/2024:02:03:03] ENGINE Bus STARTED 2024-04-27T02:03:04.126 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:04.126 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:04 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.242 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.243 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:05 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:03:05.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:03:05.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:05.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:05 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:03:06.067 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled alertmanager update... 2024-04-27T02:03:06.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:06 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:03:06.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:06 smithi151 ceph-mon[22663]: Deploying daemon prometheus.a on smithi151 2024-04-27T02:03:06.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:03:06.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[26568]: Deploying daemon prometheus.a on smithi151 2024-04-27T02:03:06.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:03:06.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[21867]: Deploying daemon prometheus.a on smithi151 2024-04-27T02:03:06.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:06.772 DEBUG:teuthology.orchestra.run.smithi026:alertmanager.a> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-27T02:03:06.776 INFO:tasks.cephadm:Adding grafana.a on smithi151 2024-04-27T02:03:06.776 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi151=a' 2024-04-27T02:03:07.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:07 smithi151 ceph-mon[22663]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:07.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:07 smithi151 ceph-mon[22663]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:07.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:07 smithi151 ceph-mon[22663]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-04-27T02:03:07.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:07 smithi026 ceph-mon[26568]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:07.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:07 smithi026 ceph-mon[26568]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:07.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:07 smithi026 ceph-mon[26568]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-04-27T02:03:07.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:07 smithi026 ceph-mon[21867]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:07.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:07 smithi026 ceph-mon[21867]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:07.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:07 smithi026 ceph-mon[21867]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-04-27T02:03:09.351 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled grafana update... 2024-04-27T02:03:09.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:09 smithi026 ceph-mon[21867]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:09.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:09 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:09.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:09 smithi026 ceph-mon[26568]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:09.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:09 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:09.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:09 smithi151 ceph-mon[22663]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:09.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:09 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:10.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:10 smithi151 ceph-mon[22663]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:10.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:10 smithi151 ceph-mon[22663]: Saving service grafana spec with placement smithi151=a;count:1 2024-04-27T02:03:10.745 DEBUG:teuthology.orchestra.run.smithi151:grafana.a> sudo journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service 2024-04-27T02:03:10.747 INFO:tasks.cephadm:Setting up client nodes... 2024-04-27T02:03:10.747 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-27T02:03:10.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:10 smithi026 ceph-mon[21867]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:10.773 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:10 smithi026 ceph-mon[21867]: Saving service grafana spec with placement smithi151=a;count:1 2024-04-27T02:03:10.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:10 smithi026 ceph-mon[26568]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:03:10.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:10 smithi026 ceph-mon[26568]: Saving service grafana spec with placement smithi151=a;count:1 2024-04-27T02:03:11.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:11 smithi151 ceph-mon[22663]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:11.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:11 smithi026 ceph-mon[21867]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:11.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:11 smithi026 ceph-mon[26568]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:12.808 INFO:teuthology.orchestra.run.smithi026.stdout:[client.0] 2024-04-27T02:03:12.808 INFO:teuthology.orchestra.run.smithi026.stdout: key = AQDgXCxm6l0oMRAAOe+lPzoS+kQp+lSiQU55cw== 2024-04-27T02:03:13.470 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:13 smithi026 ceph-mon[21867]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:13.470 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:13 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3471563073' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:13.470 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:13 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3471563073' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-27T02:03:13.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:13 smithi026 ceph-mon[26568]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:13.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:13 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3471563073' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:13.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:13 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3471563073' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-27T02:03:13.494 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-27T02:03:13.494 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-27T02:03:13.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-27T02:03:13.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-27T02:03:13.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:13 smithi151 ceph-mon[22663]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:13.688 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:13 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3471563073' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:13.688 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:13 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3471563073' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-27T02:03:15.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:15 smithi151 ceph-mon[22663]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:15.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:15 smithi026 ceph-mon[21867]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:15.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:15 smithi026 ceph-mon[26568]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:17.271 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:17 smithi151 systemd[1]: Starting Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:03:17.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:17 smithi151 ceph-mon[22663]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:17.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:17 smithi026 ceph-mon[21867]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:17.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:17 smithi026 ceph-mon[26568]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:18.100 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 podman[37758]: 2024-04-27 02:03:18.070304496 +0000 UTC m=+0.115256367 container create a5b99015bed7e1bdd59bbb512a39760211790aeec56c0e50e9acfbe6efddc2f6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:03:18.100 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 podman[37758]: 2024-04-27 02:03:17.974916646 +0000 UTC m=+0.019868524 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-27T02:03:18.103 INFO:teuthology.orchestra.run.smithi151.stdout:[client.1] 2024-04-27T02:03:18.103 INFO:teuthology.orchestra.run.smithi151.stdout: key = AQDmXCxmcYIeBxAAd9cFGL+R0FTUY2/j32cNfg== 2024-04-27T02:03:18.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:18 smithi026 ceph-mon[26568]: from='client.? 172.21.15.151:0/732126449' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:18.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:18 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:18.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:18 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-27T02:03:18.655 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:18 smithi026 ceph-mon[21867]: from='client.? 172.21.15.151:0/732126449' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:18.655 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:18 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:18.655 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:18 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-27T02:03:18.714 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-mon[22663]: from='client.? 172.21.15.151:0/732126449' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:18.714 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-27T02:03:18.714 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-27T02:03:18.714 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 podman[37758]: 2024-04-27 02:03:18.498752703 +0000 UTC m=+0.543704575 container init a5b99015bed7e1bdd59bbb512a39760211790aeec56c0e50e9acfbe6efddc2f6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 podman[37758]: 2024-04-27 02:03:18.501477142 +0000 UTC m=+0.546429024 container start a5b99015bed7e1bdd59bbb512a39760211790aeec56c0e50e9acfbe6efddc2f6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.525Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.525Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.525Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.525Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi151 (none))" 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.525Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.525Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.526Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.527Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.527Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.529Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.529Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.53µs 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.529Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.529Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-27T02:03:18.715 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.529Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.169µs wal_replay_duration=273.116µs total_replay_duration=304.876µs 2024-04-27T02:03:18.716 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.530Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-27T02:03:18.716 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.530Z caller=main.go:947 level=info msg="TSDB started" 2024-04-27T02:03:18.716 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.530Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-27T02:03:18.716 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.544Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.893459ms db_storage=634ns remote_storage=1.859µs web_handler=236ns query_engine=660ns scrape=323.484µs scrape_sd=15.071µs notify=853ns notify_sd=1.28µs rules=14.399792ms 2024-04-27T02:03:18.716 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:03:18.544Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-27T02:03:18.716 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 bash[37758]: a5b99015bed7e1bdd59bbb512a39760211790aeec56c0e50e9acfbe6efddc2f6 2024-04-27T02:03:18.716 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:03:18 smithi151 systemd[1]: Started Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:03:19.038 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-27T02:03:19.038 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-27T02:03:19.038 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-27T02:03:19.074 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-27T02:03:19.074 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-27T02:03:19.074 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-27T02:03:19.321 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:19.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:19 smithi026 ceph-mon[26568]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:19.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:19 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:19.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:19 smithi026 ceph-mon[26568]: Deploying daemon node-exporter.a on smithi026 2024-04-27T02:03:19.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:19 smithi026 ceph-mon[21867]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:19.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:19 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:19.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:19 smithi026 ceph-mon[21867]: Deploying daemon node-exporter.a on smithi026 2024-04-27T02:03:20.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:19 smithi151 ceph-mon[22663]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:20.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:19 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:20.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:19 smithi151 ceph-mon[22663]: Deploying daemon node-exporter.a on smithi026 2024-04-27T02:03:21.193 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:03:21.902 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:21 smithi151 ceph-mon[22663]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:21.902 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/61421194' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:03:22.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:21 smithi026 ceph-mon[21867]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:22.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:21 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/61421194' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:03:22.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:21 smithi026 ceph-mon[26568]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:22.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:21 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/61421194' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:03:22.306 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":20,"active_gid":24337,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":570608856},{"type":"v1","addr":"172.21.15.26:6801","nonce":570608856}]},"active_addr":"172.21.15.26:6801/570608856","active_change":"2024-04-27T02:03:01.260395+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24335,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":820053206}]},{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2500712321}]},{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2781242146}]},{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":438351353}]}]}} 2024-04-27T02:03:22.312 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-27T02:03:22.312 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-27T02:03:22.313 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-27T02:03:23.024 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:23.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:23 smithi151 ceph-mon[22663]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:23.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:23 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:23.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:23 smithi026 ceph-mon[26568]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:23.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:23 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:23.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:23 smithi026 ceph-mon[21867]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:23.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:23 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:24.099 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:23 smithi026 podman[41460]: 2024-04-27 02:03:23.912822682 +0000 UTC m=+3.415470600 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-27T02:03:24.100 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 podman[41460]: 2024-04-27 02:03:24.119945303 +0000 UTC m=+3.622593208 container create 5405a3c0085a28129ef782a9e79ffc6994e52e866163c8ee19ed307a226ceda3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 podman[41460]: 2024-04-27 02:03:24.78328239 +0000 UTC m=+4.285930296 container init 5405a3c0085a28129ef782a9e79ffc6994e52e866163c8ee19ed307a226ceda3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 podman[41460]: 2024-04-27 02:03:24.787463464 +0000 UTC m=+4.290111380 container start 5405a3c0085a28129ef782a9e79ffc6994e52e866163c8ee19ed307a226ceda3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.792Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.792Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=arp 2024-04-27T02:03:24.859 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-27T02:03:24.860 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=edac 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-27T02:03:24.861 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=os 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=stat 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-27T02:03:24.862 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=time 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=uname 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.793Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[41639]: ts=2024-04-27T02:03:24.794Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 bash[41460]: 5405a3c0085a28129ef782a9e79ffc6994e52e866163c8ee19ed307a226ceda3 2024-04-27T02:03:24.863 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:03:24 smithi026 systemd[1]: Started Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:03:25.431 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:03:25.432 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":59,"fsid":"7a5462be-0439-11ef-bc93-c7b262605968","created":"2024-04-27T01:57:47.310360+0000","modified":"2024-04-27T02:03:01.260065+0000","last_up_change":"2024-04-27T02:02:42.369042+0000","last_in_change":"2024-04-27T02:02:24.790754+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-27T02:00:46.196957+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5b7a48d3-2b4e-4b09-a4e7-819387c92ab3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6803","nonce":652787281}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6805","nonce":652787281}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6809","nonce":652787281}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6807","nonce":652787281}]},"public_addr":"172.21.15.26:6803/652787281","cluster_addr":"172.21.15.26:6805/652787281","heartbeat_back_addr":"172.21.15.26:6809/652787281","heartbeat_front_addr":"172.21.15.26:6807/652787281","state":["exists","up"]},{"osd":1,"uuid":"c9ce276b-a815-42f7-817b-2af147f9f597","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6811","nonce":1040402840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6813","nonce":1040402840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6817","nonce":1040402840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6815","nonce":1040402840}]},"public_addr":"172.21.15.26:6811/1040402840","cluster_addr":"172.21.15.26:6813/1040402840","heartbeat_back_addr":"172.21.15.26:6817/1040402840","heartbeat_front_addr":"172.21.15.26:6815/1040402840","state":["exists","up"]},{"osd":2,"uuid":"e22a75e4-af45-43bb-bf21-ab69c2046a4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6819","nonce":1403075570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6821","nonce":1403075570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6825","nonce":1403075570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6823","nonce":1403075570}]},"public_addr":"172.21.15.26:6819/1403075570","cluster_addr":"172.21.15.26:6821/1403075570","heartbeat_back_addr":"172.21.15.26:6825/1403075570","heartbeat_front_addr":"172.21.15.26:6823/1403075570","state":["exists","up"]},{"osd":3,"uuid":"a048a9f8-c1a3-4da7-908c-175069e4e4cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6827","nonce":1737241826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6829","nonce":1737241826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6833","nonce":1737241826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6831","nonce":1737241826}]},"public_addr":"172.21.15.26:6827/1737241826","cluster_addr":"172.21.15.26:6829/1737241826","heartbeat_back_addr":"172.21.15.26:6833/1737241826","heartbeat_front_addr":"172.21.15.26:6831/1737241826","state":["exists","up"]},{"osd":4,"uuid":"92076b2a-a5ba-4978-ab85-6b87fa05723f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6801","nonce":1744678093}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6803","nonce":1744678093}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6807","nonce":1744678093}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6805","nonce":1744678093}]},"public_addr":"172.21.15.151:6801/1744678093","cluster_addr":"172.21.15.151:6803/1744678093","heartbeat_back_addr":"172.21.15.151:6807/1744678093","heartbeat_front_addr":"172.21.15.151:6805/1744678093","state":["exists","up"]},{"osd":5,"uuid":"93422b66-336c-490e-aa55-22c06d91a888","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6809","nonce":3769734978}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6811","nonce":3769734978}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6815","nonce":3769734978}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6813","nonce":3769734978}]},"public_addr":"172.21.15.151:6809/3769734978","cluster_addr":"172.21.15.151:6811/3769734978","heartbeat_back_addr":"172.21.15.151:6815/3769734978","heartbeat_front_addr":"172.21.15.151:6813/3769734978","state":["exists","up"]},{"osd":6,"uuid":"2ffe8da7-1406-4415-a422-bd87364d204a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6817","nonce":2406497847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6819","nonce":2406497847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6823","nonce":2406497847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6821","nonce":2406497847}]},"public_addr":"172.21.15.151:6817/2406497847","cluster_addr":"172.21.15.151:6819/2406497847","heartbeat_back_addr":"172.21.15.151:6823/2406497847","heartbeat_front_addr":"172.21.15.151:6821/2406497847","state":["exists","up"]},{"osd":7,"uuid":"ef6a01df-7c83-4775-a7c7-7f85b885ac12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6825","nonce":3440470593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6827","nonce":3440470593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6831","nonce":3440470593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6829","nonce":3440470593}]},"public_addr":"172.21.15.151:6825/3440470593","cluster_addr":"172.21.15.151:6827/3440470593","heartbeat_back_addr":"172.21.15.151:6831/3440470593","heartbeat_front_addr":"172.21.15.151:6829/3440470593","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T01:59:57.301422+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:00:20.149531+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:00:43.693071+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:09.029421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:30.542566+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:52.602062+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:02:16.204412+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:02:40.427748+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/2363584941":"2024-04-28T02:03:01.260033+0000","172.21.15.26:6800/1966443158":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/930416246":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/1866163037":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/124039062":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6801/1966443158":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/384921090":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/358798680":"2024-04-28T01:58:41.640278+0000","172.21.15.26:0/102542096":"2024-04-28T01:58:19.403412+0000","172.21.15.26:0/2505340537":"2024-04-28T01:58:19.403412+0000","172.21.15.26:6800/518236597":"2024-04-28T01:58:19.403412+0000","172.21.15.26:6801/1573366204":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6800/1573366204":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6801/518236597":"2024-04-28T01:58:19.403412+0000","172.21.15.26:0/4281274133":"2024-04-28T01:58:41.640278+0000","172.21.15.26:0/1738905563":"2024-04-28T01:58:19.403412+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-27T02:03:25.850 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:25 smithi151 bash[38049]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-27T02:03:26.185 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:25 smithi151 ceph-mon[22663]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:26.185 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:25 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:26.185 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:25 smithi151 ceph-mon[22663]: Deploying daemon node-exporter.b on smithi151 2024-04-27T02:03:26.185 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:25 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3550681565' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:03:26.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[21867]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:26.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:26.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[21867]: Deploying daemon node-exporter.b on smithi151 2024-04-27T02:03:26.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3550681565' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:03:26.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[26568]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:26.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:26.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[26568]: Deploying daemon node-exporter.b on smithi151 2024-04-27T02:03:26.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:25 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3550681565' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:03:26.333 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-27T02:03:26.333 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-27T02:03:26.560 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:26.599 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:26 smithi151 bash[38049]: Getting image source signatures 2024-04-27T02:03:26.599 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:26 smithi151 bash[38049]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-27T02:03:26.599 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:26 smithi151 bash[38049]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-27T02:03:26.599 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:26 smithi151 bash[38049]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-27T02:03:27.001 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:26] "GET /metrics HTTP/1.1" 200 191500 "" "Prometheus/2.33.4" 2024-04-27T02:03:27.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:03:27.849 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:27 smithi151 bash[38049]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-27T02:03:27.849 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:27 smithi151 bash[38049]: Writing manifest to image destination 2024-04-27T02:03:27.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:27 smithi026 ceph-mon[21867]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:27.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:27 smithi026 ceph-mon[26568]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:28.221 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:27 smithi151 ceph-mon[22663]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:28.222 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 podman[38049]: 2024-04-27 02:03:27.927290006 +0000 UTC m=+2.237226945 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-27T02:03:28.222 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 podman[38049]: 2024-04-27 02:03:28.048310362 +0000 UTC m=+2.358247302 container create 8e60a39d89eda768cc4fc450eb444789e324886558dda736bc23eef9be23060d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:03:28.222 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 podman[38049]: 2024-04-27 02:03:28.242139168 +0000 UTC m=+2.552076112 container init 8e60a39d89eda768cc4fc450eb444789e324886558dda736bc23eef9be23060d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:03:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:03:28.420 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":59,"fsid":"7a5462be-0439-11ef-bc93-c7b262605968","created":"2024-04-27T01:57:47.310360+0000","modified":"2024-04-27T02:03:01.260065+0000","last_up_change":"2024-04-27T02:02:42.369042+0000","last_in_change":"2024-04-27T02:02:24.790754+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-27T02:00:46.196957+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5b7a48d3-2b4e-4b09-a4e7-819387c92ab3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6803","nonce":652787281}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6805","nonce":652787281}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6809","nonce":652787281}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":652787281},{"type":"v1","addr":"172.21.15.26:6807","nonce":652787281}]},"public_addr":"172.21.15.26:6803/652787281","cluster_addr":"172.21.15.26:6805/652787281","heartbeat_back_addr":"172.21.15.26:6809/652787281","heartbeat_front_addr":"172.21.15.26:6807/652787281","state":["exists","up"]},{"osd":1,"uuid":"c9ce276b-a815-42f7-817b-2af147f9f597","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6811","nonce":1040402840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6813","nonce":1040402840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6817","nonce":1040402840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1040402840},{"type":"v1","addr":"172.21.15.26:6815","nonce":1040402840}]},"public_addr":"172.21.15.26:6811/1040402840","cluster_addr":"172.21.15.26:6813/1040402840","heartbeat_back_addr":"172.21.15.26:6817/1040402840","heartbeat_front_addr":"172.21.15.26:6815/1040402840","state":["exists","up"]},{"osd":2,"uuid":"e22a75e4-af45-43bb-bf21-ab69c2046a4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6819","nonce":1403075570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6821","nonce":1403075570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6825","nonce":1403075570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1403075570},{"type":"v1","addr":"172.21.15.26:6823","nonce":1403075570}]},"public_addr":"172.21.15.26:6819/1403075570","cluster_addr":"172.21.15.26:6821/1403075570","heartbeat_back_addr":"172.21.15.26:6825/1403075570","heartbeat_front_addr":"172.21.15.26:6823/1403075570","state":["exists","up"]},{"osd":3,"uuid":"a048a9f8-c1a3-4da7-908c-175069e4e4cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6827","nonce":1737241826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6829","nonce":1737241826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6833","nonce":1737241826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1737241826},{"type":"v1","addr":"172.21.15.26:6831","nonce":1737241826}]},"public_addr":"172.21.15.26:6827/1737241826","cluster_addr":"172.21.15.26:6829/1737241826","heartbeat_back_addr":"172.21.15.26:6833/1737241826","heartbeat_front_addr":"172.21.15.26:6831/1737241826","state":["exists","up"]},{"osd":4,"uuid":"92076b2a-a5ba-4978-ab85-6b87fa05723f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6801","nonce":1744678093}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6803","nonce":1744678093}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6807","nonce":1744678093}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1744678093},{"type":"v1","addr":"172.21.15.151:6805","nonce":1744678093}]},"public_addr":"172.21.15.151:6801/1744678093","cluster_addr":"172.21.15.151:6803/1744678093","heartbeat_back_addr":"172.21.15.151:6807/1744678093","heartbeat_front_addr":"172.21.15.151:6805/1744678093","state":["exists","up"]},{"osd":5,"uuid":"93422b66-336c-490e-aa55-22c06d91a888","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6809","nonce":3769734978}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6811","nonce":3769734978}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6815","nonce":3769734978}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":3769734978},{"type":"v1","addr":"172.21.15.151:6813","nonce":3769734978}]},"public_addr":"172.21.15.151:6809/3769734978","cluster_addr":"172.21.15.151:6811/3769734978","heartbeat_back_addr":"172.21.15.151:6815/3769734978","heartbeat_front_addr":"172.21.15.151:6813/3769734978","state":["exists","up"]},{"osd":6,"uuid":"2ffe8da7-1406-4415-a422-bd87364d204a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6817","nonce":2406497847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6819","nonce":2406497847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6823","nonce":2406497847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2406497847},{"type":"v1","addr":"172.21.15.151:6821","nonce":2406497847}]},"public_addr":"172.21.15.151:6817/2406497847","cluster_addr":"172.21.15.151:6819/2406497847","heartbeat_back_addr":"172.21.15.151:6823/2406497847","heartbeat_front_addr":"172.21.15.151:6821/2406497847","state":["exists","up"]},{"osd":7,"uuid":"ef6a01df-7c83-4775-a7c7-7f85b885ac12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6825","nonce":3440470593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6827","nonce":3440470593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6831","nonce":3440470593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":3440470593},{"type":"v1","addr":"172.21.15.151:6829","nonce":3440470593}]},"public_addr":"172.21.15.151:6825/3440470593","cluster_addr":"172.21.15.151:6827/3440470593","heartbeat_back_addr":"172.21.15.151:6831/3440470593","heartbeat_front_addr":"172.21.15.151:6829/3440470593","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T01:59:57.301422+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:00:20.149531+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:00:43.693071+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:09.029421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:30.542566+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:01:52.602062+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:02:16.204412+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-27T02:02:40.427748+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/2363584941":"2024-04-28T02:03:01.260033+0000","172.21.15.26:6800/1966443158":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/930416246":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/1866163037":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/124039062":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6801/1966443158":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/384921090":"2024-04-28T02:03:01.260033+0000","172.21.15.26:0/358798680":"2024-04-28T01:58:41.640278+0000","172.21.15.26:0/102542096":"2024-04-28T01:58:19.403412+0000","172.21.15.26:0/2505340537":"2024-04-28T01:58:19.403412+0000","172.21.15.26:6800/518236597":"2024-04-28T01:58:19.403412+0000","172.21.15.26:6801/1573366204":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6800/1573366204":"2024-04-28T01:58:41.640278+0000","172.21.15.26:6801/518236597":"2024-04-28T01:58:19.403412+0000","172.21.15.26:0/4281274133":"2024-04-28T01:58:41.640278+0000","172.21.15.26:0/1738905563":"2024-04-28T01:58:19.403412+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-27T02:03:28.600 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 podman[38049]: 2024-04-27 02:03:28.245579134 +0000 UTC m=+2.555516076 container start 8e60a39d89eda768cc4fc450eb444789e324886558dda736bc23eef9be23060d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:03:28.600 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.250Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-27T02:03:28.600 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.250Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-27T02:03:28.600 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.250Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-27T02:03:28.600 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.250Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-27T02:03:28.600 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=arp 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=edac 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-27T02:03:28.601 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-27T02:03:28.602 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=os 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=stat 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=time 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-27T02:03:28.603 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=uname 2024-04-27T02:03:28.604 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-27T02:03:28.604 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-27T02:03:28.604 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-27T02:03:28.604 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-27T02:03:28.604 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[38100]: ts=2024-04-27T02:03:28.251Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-27T02:03:28.604 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 bash[38049]: 8e60a39d89eda768cc4fc450eb444789e324886558dda736bc23eef9be23060d 2024-04-27T02:03:28.604 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:03:28 smithi151 systemd[1]: Started Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:03:29.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-27T02:03:29.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-27T02:03:29.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-27T02:03:29.084 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-27T02:03:29.084 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-27T02:03:29.084 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-27T02:03:29.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-27T02:03:29.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-27T02:03:29.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[21867]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:29.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:29.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3540795327' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[26568]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:03:29.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:29 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3540795327' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:03:29.813 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:29 smithi151 ceph-mon[22663]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:29.814 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:29 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:29.814 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:29 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:03:29.814 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:29 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:03:29.814 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:29 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:03:29.814 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:29 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3540795327' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-27T02:03:29.910 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:30.601 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:31.168 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:31.169 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:31.170 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:31.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:31 smithi026 ceph-mon[21867]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:31.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:31 smithi026 ceph-mon[26568]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:31.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:31 smithi151 ceph-mon[22663]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:33.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:33 smithi026 ceph-mon[21867]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:33.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:33 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:33.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:33 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:33.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:33 smithi026 ceph-mon[26568]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:33.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:33 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:33.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:33 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:33.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:33 smithi151 ceph-mon[22663]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:33.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:33 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:33.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:33 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:35.122 INFO:teuthology.orchestra.run.smithi026.stdout:197568495632 2024-04-27T02:03:35.122 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-27T02:03:35.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:35 smithi151 ceph-mon[22663]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:35.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:35 smithi026 ceph-mon[21867]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:35.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:35 smithi026 ceph-mon[26568]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:36.528 INFO:teuthology.orchestra.run.smithi026.stdout:163208757269 2024-04-27T02:03:36.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-27T02:03:37.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:37 smithi026 ceph-mon[21867]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:37.202 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:36] "GET /metrics HTTP/1.1" 200 191488 "" "Prometheus/2.33.4" 2024-04-27T02:03:37.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:37 smithi026 ceph-mon[26568]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:37.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:37 smithi151 ceph-mon[22663]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:37.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:03:39.389 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:39.391 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:39.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:39 smithi026 ceph-mon[21867]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:39.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:39 smithi026 ceph-mon[26568]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:39.716 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:39.716 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:39.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:39 smithi151 ceph-mon[22663]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout:77309411364 2024-04-27T02:03:40.571 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-27T02:03:41.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:41 smithi151 ceph-mon[22663]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:41.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:41 smithi026 ceph-mon[21867]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:41.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:41 smithi026 ceph-mon[26568]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:42.865 INFO:teuthology.orchestra.run.smithi026.stdout:128849018907 2024-04-27T02:03:42.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-27T02:03:43.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:43 smithi026 ceph-mon[21867]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:43.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:43 smithi026 ceph-mon[26568]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:43.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:43 smithi151 ceph-mon[22663]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:45.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:45.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:45.110 INFO:teuthology.orchestra.run.smithi026.stdout:197568495634 2024-04-27T02:03:45.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[21867]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:45.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1833563734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-27T02:03:45.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:45.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:45.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[26568]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:45.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1833563734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-27T02:03:45.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:45.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:45 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:45.664 INFO:teuthology.orchestra.run.smithi026.stdout:55834574889 2024-04-27T02:03:45.665 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-27T02:03:45.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:45 smithi151 ceph-mon[22663]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:45.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:45 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1833563734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-27T02:03:45.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:45 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:45.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:45 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:03:46.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:46 smithi026 ceph-mon[21867]: Deploying daemon alertmanager.a on smithi026 2024-04-27T02:03:46.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:46 smithi026 ceph-mon[26568]: Deploying daemon alertmanager.a on smithi026 2024-04-27T02:03:46.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:46 smithi151 ceph-mon[22663]: Deploying daemon alertmanager.a on smithi026 2024-04-27T02:03:47.147 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:46] "GET /metrics HTTP/1.1" 200 191488 "" "Prometheus/2.33.4" 2024-04-27T02:03:47.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:03:47.444 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:47 smithi026 ceph-mon[21867]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:47.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:47 smithi026 ceph-mon[26568]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:47.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:47 smithi151 ceph-mon[22663]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:48.398 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495632 got 197568495634 for osd.6 2024-04-27T02:03:48.398 DEBUG:teuthology.parallel:result is None 2024-04-27T02:03:48.415 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:48.415 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:49.133 INFO:teuthology.orchestra.run.smithi026.stdout:107374182432 2024-04-27T02:03:49.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-27T02:03:49.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:49 smithi026 ceph-mon[21867]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:49.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:49 smithi026 ceph-mon[26568]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:49.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:49 smithi151 ceph-mon[22663]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:50.392 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:03:51.651 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:51 smithi151 ceph-mon[22663]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:51.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:51 smithi026 ceph-mon[21867]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:51.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:51 smithi026 ceph-mon[26568]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:53.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:53 smithi026 ceph-mon[21867]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:53.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:53 smithi026 ceph-mon[26568]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:53.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:53 smithi151 ceph-mon[22663]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:54.106 INFO:teuthology.orchestra.run.smithi026.stdout:34359738416 2024-04-27T02:03:54.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-27T02:03:55.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:55 smithi026 ceph-mon[21867]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:55.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:55 smithi026 ceph-mon[26568]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:55.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:55 smithi151 ceph-mon[22663]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:56.095 INFO:teuthology.orchestra.run.smithi026.stdout:163208757273 2024-04-27T02:03:56.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:56 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1463012135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-27T02:03:56.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:56 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1463012135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-27T02:03:56.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:56 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1463012135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-27T02:03:57.041 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:03:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:56] "GET /metrics HTTP/1.1" 200 191475 "" "Prometheus/2.33.4" 2024-04-27T02:03:57.092 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757269 got 163208757273 for osd.5 2024-04-27T02:03:57.093 DEBUG:teuthology.parallel:result is None 2024-04-27T02:03:57.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:03:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:03:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:03:57.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:57 smithi026 ceph-mon[21867]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:57.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:57 smithi026 ceph-mon[26568]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:57.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:57 smithi151 ceph-mon[22663]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:58.362 INFO:teuthology.orchestra.run.smithi026.stdout:77309411367 2024-04-27T02:03:58.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:58 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1152270651' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-27T02:03:58.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:58 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1152270651' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-27T02:03:58.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:58 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1152270651' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-27T02:03:59.227 INFO:teuthology.orchestra.run.smithi026.stdout:55834574892 2024-04-27T02:03:59.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:59 smithi026 ceph-mon[26568]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:59.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:03:59 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2824716846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-27T02:03:59.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:59 smithi026 ceph-mon[21867]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:59.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:03:59 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2824716846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-27T02:03:59.671 INFO:teuthology.orchestra.run.smithi026.stdout:231928234001 2024-04-27T02:03:59.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-27T02:03:59.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:59 smithi151 ceph-mon[22663]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:03:59.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:03:59 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2824716846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-27T02:04:00.376 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574892 for osd.1 2024-04-27T02:04:00.377 DEBUG:teuthology.parallel:result is None 2024-04-27T02:04:01.004 INFO:teuthology.orchestra.run.smithi026.stdout:128849018911 2024-04-27T02:04:01.257 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:04:01.257 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:04:01.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:01 smithi026 ceph-mon[21867]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:01.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:01 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2349221606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-27T02:04:01.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:01 smithi026 ceph-mon[26568]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:01.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:01 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2349221606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-27T02:04:01.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:01 smithi151 ceph-mon[22663]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:01.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:01 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2349221606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-27T02:04:02.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:04:02.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:04:02.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:04:02.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:04:02.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:04:02.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:04:02.753 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411364 got 77309411367 for osd.2 2024-04-27T02:04:02.754 DEBUG:teuthology.parallel:result is None 2024-04-27T02:04:03.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:03 smithi026 ceph-mon[21867]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:03.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:03 smithi026 ceph-mon[26568]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:03.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:03 smithi151 ceph-mon[22663]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:05.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:05 smithi026 ceph-mon[21867]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:05.554 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:05 smithi026 ceph-mon[26568]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:05.848 INFO:teuthology.orchestra.run.smithi026.stdout:107374182436 2024-04-27T02:04:05.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:05 smithi151 ceph-mon[22663]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:06.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:06 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4125173188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-27T02:04:06.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:06 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4125173188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-27T02:04:06.797 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:06 smithi026 systemd[1]: Starting Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:04:06.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:06 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4125173188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-27T02:04:07.062 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:04:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:06] "GET /metrics HTTP/1.1" 200 191479 "" "Prometheus/2.33.4" 2024-04-27T02:04:07.063 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018911 for osd.4 2024-04-27T02:04:07.063 DEBUG:teuthology.parallel:result is None 2024-04-27T02:04:07.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:04:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:04:07.479 INFO:teuthology.orchestra.run.smithi026.stdout:231928234002 2024-04-27T02:04:07.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:07 smithi026 ceph-mon[21867]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:07.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:07 smithi026 ceph-mon[26568]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:07.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:07 smithi151 ceph-mon[22663]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:08.515 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182436 for osd.3 2024-04-27T02:04:08.515 DEBUG:teuthology.parallel:result is None 2024-04-27T02:04:08.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:08 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/278671245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-27T02:04:08.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:08 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/278671245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-27T02:04:08.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:08 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/278671245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-27T02:04:08.919 INFO:teuthology.orchestra.run.smithi026.stdout:34359738419 2024-04-27T02:04:09.091 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:08 smithi026 podman[44689]: 2024-04-27 02:04:08.756675357 +0000 UTC m=+0.203849620 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-27T02:04:09.115 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234001 got 231928234002 for osd.7 2024-04-27T02:04:09.116 DEBUG:teuthology.parallel:result is None 2024-04-27T02:04:09.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-mon[21867]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:09.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3552642368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-27T02:04:09.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-mon[26568]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:09.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3552642368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-27T02:04:09.743 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 podman[44689]: 2024-04-27 02:04:09.500683019 +0000 UTC m=+0.947857269 volume create 00e8b7b115a02e94513bae6f6c7fd21c3c36dad9c5fc75d9bd252f4cc60c94fd 2024-04-27T02:04:09.743 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 podman[44689]: 2024-04-27 02:04:09.597262748 +0000 UTC m=+1.044437040 container create 50d1bfe425cce0f8231c696c5c16b5e78f24fd505d4b37e68b7e1e2ef4b9d1c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:09.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:09 smithi151 ceph-mon[22663]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:09.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:09 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3552642368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 podman[44689]: 2024-04-27 02:04:09.858317529 +0000 UTC m=+1.305491796 container init 50d1bfe425cce0f8231c696c5c16b5e78f24fd505d4b37e68b7e1e2ef4b9d1c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 podman[44689]: 2024-04-27 02:04:09.861426473 +0000 UTC m=+1.308600733 container start 50d1bfe425cce0f8231c696c5c16b5e78f24fd505d4b37e68b7e1e2ef4b9d1c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.873Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.874Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.874Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.876Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.904Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.904Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.907Z caller=main.go:518 msg=Listening address=:9093 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:09.907Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-27T02:04:10.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 bash[44689]: 50d1bfe425cce0f8231c696c5c16b5e78f24fd505d4b37e68b7e1e2ef4b9d1c6 2024-04-27T02:04:10.110 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:09 smithi026 systemd[1]: Started Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:04:10.440 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738419 for osd.0 2024-04-27T02:04:10.440 DEBUG:teuthology.parallel:result is None 2024-04-27T02:04:10.441 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-27T02:04:10.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-27T02:04:10.664 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:04:11.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:11.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[21867]: Deploying daemon grafana.a on smithi151 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.078 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-mon[26568]: Deploying daemon grafana.a on smithi151 2024-04-27T02:04:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:04:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:04:11.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:11.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:11 smithi151 ceph-mon[22663]: Deploying daemon grafana.a on smithi151 2024-04-27T02:04:12.108 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:11 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:11.877Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000582122s 2024-04-27T02:04:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:04:12.562 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-04-27T02:04:13.175 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-27T02:04:12.018479+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48848,"kb_used_data":6096,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912368,"statfs":{"total":767960285184,"available":767910264832,"internally_reserved":0,"allocated":6242304,"data_stored":3488982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002509"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-27T02:03:02.095248+0000","last_change":"2024-04-27T02:02:46.673424+0000","last_active":"2024-04-27T02:03:02.095248+0000","last_peered":"2024-04-27T02:03:02.095248+0000","last_clean":"2024-04-27T02:03:02.095248+0000","last_became_active":"2024-04-27T02:02:46.673021+0000","last_became_peered":"2024-04-27T02:02:46.673021+0000","last_unstale":"2024-04-27T02:03:02.095248+0000","last_undegraded":"2024-04-27T02:03:02.095248+0000","last_fullsized":"2024-04-27T02:03:02.095248+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-27T02:00:46.676908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-27T02:00:46.676908+0000","last_clean_scrub_stamp":"2024-04-27T02:00:46.676908+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-28T10:03:15.589002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.996,"5min":0.996,"15min":0.996},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.5}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.9789999999999992,"5min":9.9789999999999992,"15min":9.9789999999999992},"last":0.56000000000000005}]},{"osd":1,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.003,"5min":10.003,"15min":10.003},"last":0.66200000000000003}]},{"osd":2,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.013999999999999,"5min":10.013999999999999,"15min":10.013999999999999},"last":0.59899999999999998}]},{"osd":3,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.9689999999999994,"5min":9.9689999999999994,"15min":9.9689999999999994},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.61899999999999999}]},{"osd":4,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.437}]},{"osd":7,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.49099999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:29 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.35099999999999998},"min":{"1min":0.224,"5min":0.189,"15min":0.189},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.42099999999999999,"15min":0.40300000000000002},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.34899999999999998}]},{"osd":2,"last update":"Sat Apr 27 02:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.39400000000000002,"15min":0.39900000000000002},"min":{"1min":0.23999999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.52000000000000002,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.41499999999999998,"15min":0.41299999999999998},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.67100000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Apr 27 02:03:14 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.59799999999999998,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Apr 27 02:03:37 2024","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":0.71999999999999997,"15min":0.63800000000000001},"min":{"1min":0.42799999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":12.444000000000001,"5min":12.444000000000001,"15min":12.444000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Apr 27 02:03:57 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.629,"15min":0.63},"min":{"1min":0.501,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79600000000000004,"5min":0.871,"15min":0.871},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.623,"15min":0.625},"min":{"1min":0.39200000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Apr 27 02:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.62},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.55700000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 27 02:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.374,"15min":0.35899999999999999},"min":{"1min":0.26900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.38700000000000001,"15min":0.36099999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Apr 27 02:03:51 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.42099999999999999,"15min":0.40600000000000003},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.378},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.42699999999999999,"15min":0.41899999999999998},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.57999999999999996,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Apr 27 02:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.50900000000000001,"15min":0.51200000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.50700000000000001,"15min":0.51300000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69699999999999995,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.434}]},{"osd":4,"last update":"Sat Apr 27 02:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.61699999999999999,"15min":0.628},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.71899999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.64900000000000002,"15min":0.66100000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72699999999999998,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.48199999999999998}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.61699999999999999,"15min":0.626},"min":{"1min":0.42899999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.78600000000000003,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.61799999999999999}]},{"osd":6,"last update":"Sat Apr 27 02:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.498}]},{"osd":7,"last update":"Sat Apr 27 02:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.57599999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6208,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738944,"statfs":{"total":95995035648,"available":95988678656,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39700000000000002,"15min":0.38500000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54000000000000004,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42699999999999999,"15min":0.41399999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.55900000000000005,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Apr 27 02:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41899999999999998,"15min":0.41099999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.57399999999999995,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.46500000000000002,"15min":0.46100000000000002},"min":{"1min":0.29899999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.63200000000000001,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.51200000000000001,"15min":0.51500000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.48099999999999998,"15min":0.48599999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.50800000000000001}]},{"osd":4,"last update":"Sat Apr 27 02:03:39 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.625},{"interface":"front","average":{"1min":0.621,"5min":0.627,"15min":0.628},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.66100000000000003,"15min":0.67000000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.76200000000000001,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.64700000000000002,"15min":0.65500000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83199999999999996,"5min":1.018,"15min":1.018},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Apr 27 02:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.68300000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:12 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42999999999999999,"15min":0.435},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57799999999999996,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.435,"5min":0.44,"15min":0.441},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.38800000000000001}]},{"osd":1,"last update":"Sat Apr 27 02:03:12 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.33000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.59199999999999997,"5min":0.623,"15min":0.623},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.501,"15min":0.505},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.60499999999999998,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.35599999999999998}]},{"osd":2,"last update":"Sat Apr 27 02:03:12 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.498,"15min":0.502},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61199999999999999,"5min":0.627,"15min":0.627},"last":0.308},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.48699999999999999,"15min":0.48899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76200000000000001,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.63,"15min":0.63},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.63600000000000001,"15min":0.64100000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82899999999999996,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.63600000000000001,"15min":0.63300000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.93799999999999994,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.64300000000000002,"15min":0.63700000000000001},"min":{"1min":0.47799999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.877,"5min":0.879,"15min":0.879},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.501}]},{"osd":7,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.51500000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.69899999999999995,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.47999999999999998}]},{"osd":1,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.39500000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.437},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.38600000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.73499999999999999,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57199999999999995,"15min":0.57699999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76300000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":0.55000000000000004}]},{"osd":3,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.41299999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57299999999999995,"15min":0.57199999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.57099999999999995}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.41099999999999998,"15min":0.39800000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.439,"15min":0.42899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.622}]},{"osd":6,"last update":"Sat Apr 27 02:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Apr 27 02:03:42 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":0.496},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.4710000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.63500000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.65900000000000003,"15min":0.57599999999999996},"min":{"1min":0.40000000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":15,"5min":15,"15min":15},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.75800000000000001,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.51600000000000001}]},{"osd":1,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78400000000000003,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57999999999999996,"15min":0.58099999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79500000000000004,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69299999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58099999999999996,"15min":0.57799999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76500000000000001,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.59699999999999998}]},{"osd":4,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.40500000000000003,"15min":0.39900000000000002},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.42099999999999999,"15min":0.41699999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.63200000000000001}]},{"osd":7,"last update":"Sat Apr 27 02:03:42 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.58899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-27T02:04:13.176 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-27T02:04:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:13 smithi151 ceph-mon[22663]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:13 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:13 smithi151 ceph-mon[22663]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-27T02:04:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:13 smithi026 ceph-mon[21867]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:13 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:13 smithi026 ceph-mon[21867]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-27T02:04:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:13 smithi026 ceph-mon[26568]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:13 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:13 smithi026 ceph-mon[26568]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-27T02:04:13.399 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:04:15.084 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:04:15.085 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-04-27T02:04:15.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:15 smithi026 ceph-mon[21867]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:15.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:15 smithi026 ceph-mon[26568]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:15.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:15 smithi151 ceph-mon[22663]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:15.720 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-04-27T02:04:14.019051+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48848,"kb_used_data":6096,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912368,"statfs":{"total":767960285184,"available":767910264832,"internally_reserved":0,"allocated":6242304,"data_stored":3488982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002787"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-27T02:03:02.095248+0000","last_change":"2024-04-27T02:02:46.673424+0000","last_active":"2024-04-27T02:03:02.095248+0000","last_peered":"2024-04-27T02:03:02.095248+0000","last_clean":"2024-04-27T02:03:02.095248+0000","last_became_active":"2024-04-27T02:02:46.673021+0000","last_became_peered":"2024-04-27T02:02:46.673021+0000","last_unstale":"2024-04-27T02:03:02.095248+0000","last_undegraded":"2024-04-27T02:03:02.095248+0000","last_fullsized":"2024-04-27T02:03:02.095248+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-27T02:00:46.676908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-27T02:00:46.676908+0000","last_clean_scrub_stamp":"2024-04-27T02:00:46.676908+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-28T10:03:15.589002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.55200000000000005}]},{"osd":1,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.53100000000000003}]},{"osd":3,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.49199999999999999}]},{"osd":4,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.312},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.996,"5min":0.996,"15min":0.996},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.34899999999999998}]},{"osd":6,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.375},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.41099999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.9789999999999992,"5min":9.9789999999999992,"15min":9.9789999999999992},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.003,"5min":10.003,"15min":10.003},"last":0.34899999999999998}]},{"osd":2,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.013999999999999,"5min":10.013999999999999,"15min":10.013999999999999},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.9689999999999994,"5min":9.9689999999999994,"15min":9.9689999999999994},"last":0.435},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.32000000000000001}]},{"osd":5,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.40999999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:29 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.35099999999999998},"min":{"1min":0.224,"5min":0.189,"15min":0.189},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.372},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.42099999999999999,"15min":0.40300000000000002},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54900000000000004}]},{"osd":2,"last update":"Sat Apr 27 02:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.39400000000000002,"15min":0.39900000000000002},"min":{"1min":0.23999999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.52000000000000002,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.41499999999999998,"15min":0.41299999999999998},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.67100000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Apr 27 02:03:14 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.59799999999999998,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Apr 27 02:03:37 2024","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":0.71999999999999997,"15min":0.63800000000000001},"min":{"1min":0.42799999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":12.444000000000001,"5min":12.444000000000001,"15min":12.444000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Apr 27 02:03:57 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.629,"15min":0.63},"min":{"1min":0.501,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79600000000000004,"5min":0.871,"15min":0.871},"last":0.752},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.623,"15min":0.625},"min":{"1min":0.39200000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.70099999999999996}]},{"osd":7,"last update":"Sat Apr 27 02:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.57799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 27 02:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.374,"15min":0.35899999999999999},"min":{"1min":0.26900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.38700000000000001,"15min":0.36099999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.52300000000000002}]},{"osd":2,"last update":"Sat Apr 27 02:03:51 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.42099999999999999,"15min":0.40600000000000003},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.42699999999999999,"15min":0.41899999999999998},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.57999999999999996,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Apr 27 02:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.50900000000000001,"15min":0.51200000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.50700000000000001,"15min":0.51300000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69699999999999995,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.46300000000000002}]},{"osd":4,"last update":"Sat Apr 27 02:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.61699999999999999,"15min":0.628},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.71899999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.64900000000000002,"15min":0.66100000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72699999999999998,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.84199999999999997}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.61699999999999999,"15min":0.626},"min":{"1min":0.42899999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.78600000000000003,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.80200000000000005}]},{"osd":6,"last update":"Sat Apr 27 02:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.73499999999999999}]},{"osd":7,"last update":"Sat Apr 27 02:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.75},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.76900000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6208,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738944,"statfs":{"total":95995035648,"available":95988678656,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39700000000000002,"15min":0.38500000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54000000000000004,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42699999999999999,"15min":0.41399999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.55900000000000005,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Apr 27 02:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41899999999999998,"15min":0.41099999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.57399999999999995,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.46500000000000002,"15min":0.46100000000000002},"min":{"1min":0.29899999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.63200000000000001,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.51200000000000001,"15min":0.51500000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.377},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.48099999999999998,"15min":0.48599999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.314}]},{"osd":4,"last update":"Sat Apr 27 02:03:39 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.627,"15min":0.628},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.66100000000000003,"15min":0.67000000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.76200000000000001,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.64700000000000002,"15min":0.65500000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83199999999999996,"5min":1.018,"15min":1.018},"last":0.439}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Apr 27 02:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.501},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.39100000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:12 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42999999999999999,"15min":0.435},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57799999999999996,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.438},{"interface":"front","average":{"1min":0.435,"5min":0.44,"15min":0.441},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.249}]},{"osd":1,"last update":"Sat Apr 27 02:03:12 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.33000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.59199999999999997,"5min":0.623,"15min":0.623},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.501,"15min":0.505},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.60499999999999998,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.28699999999999998}]},{"osd":2,"last update":"Sat Apr 27 02:03:12 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.498,"15min":0.502},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61199999999999999,"5min":0.627,"15min":0.627},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.48699999999999999,"15min":0.48899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76200000000000001,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.27100000000000002}]},{"osd":4,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.63,"15min":0.63},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.63600000000000001,"15min":0.64100000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82899999999999996,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.63600000000000001,"15min":0.63300000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.93799999999999994,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.64300000000000002,"15min":0.63700000000000001},"min":{"1min":0.47799999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.877,"5min":0.879,"15min":0.879},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Apr 27 02:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.45500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018914,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.69899999999999995,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.622},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.39500000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.38600000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.73499999999999999,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.66000000000000003}]},{"osd":2,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57199999999999995,"15min":0.57699999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76300000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Apr 27 02:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.41299999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57299999999999995,"15min":0.57199999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Apr 27 02:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.41099999999999998,"15min":0.39800000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":0.374},{"interface":"front","average":{"1min":0.499,"5min":0.439,"15min":0.42899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Apr 27 02:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Apr 27 02:03:42 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.4710000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.439}]}]},{"osd":5,"up_from":38,"seq":163208757277,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.65900000000000003,"15min":0.57599999999999996},"min":{"1min":0.40000000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":15,"5min":15,"15min":15},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.75800000000000001,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.505}]},{"osd":1,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78400000000000003,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57999999999999996,"15min":0.58099999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79500000000000004,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.51700000000000002}]},{"osd":3,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69299999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58099999999999996,"15min":0.57799999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76500000000000001,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.56399999999999995}]},{"osd":4,"last update":"Sat Apr 27 02:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.40500000000000003,"15min":0.39900000000000002},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.42099999999999999,"15min":0.41699999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.61199999999999999}]},{"osd":6,"last update":"Sat Apr 27 02:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Apr 27 02:03:42 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.437},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.55600000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-27T02:04:15.722 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-27T02:04:15.722 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-27T02:04:15.722 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-27T02:04:15.722 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-27T02:04:15.950 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/mon.a/config 2024-04-27T02:04:16.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:16 smithi026 ceph-mon[21867]: from='client.24514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-27T02:04:16.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:16 smithi026 ceph-mon[26568]: from='client.24514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-27T02:04:16.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:16 smithi151 ceph-mon[22663]: from='client.24514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-27T02:04:17.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:17 smithi026 ceph-mon[21867]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:17.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:17 smithi026 ceph-mon[26568]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:17.203 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:04:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:16] "GET /metrics HTTP/1.1" 200 191479 "" "Prometheus/2.33.4" 2024-04-27T02:04:17.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:17 smithi151 ceph-mon[22663]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:17.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:04:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:04:17.641 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:04:17.641 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-27T02:04:18.270 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-27T02:04:18.270 INFO:tasks.cephadm:Setup complete, yielding 2024-04-27T02:04:18.271 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-27T02:04:18.281 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-04-27T02:04:18.282 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-27T02:04:18.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:18 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1162715357' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-27T02:04:18.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:18 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1162715357' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-27T02:04:18.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:18 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1162715357' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-27T02:04:19.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:19 smithi026 ceph-mon[21867]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:19.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:19 smithi026 ceph-mon[26568]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:19.658 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:19 smithi151 ceph-mon[22663]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:20.108 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:19.880Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003821245s 2024-04-27T02:04:21.471 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:04:21.471 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "f69746c3-3a88-4f46-a5ee-0cbb789bc2e2", 2024-04-27T02:04:21.471 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "r", 2024-04-27T02:04:21.471 INFO:teuthology.orchestra.run.smithi026.stdout: "current_period": "dc8ecf8f-b583-4bcd-be51-610f45b84b06", 2024-04-27T02:04:21.471 INFO:teuthology.orchestra.run.smithi026.stdout: "epoch": 1 2024-04-27T02:04:21.472 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:04:21.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:21 smithi151 ceph-mon[22663]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:21.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:21 smithi151 ceph-mon[22663]: osdmap e60: 8 total, 8 up, 8 in 2024-04-27T02:04:21.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2984656555' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-27T02:04:21.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:21 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-27T02:04:21.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[21867]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:21.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[21867]: osdmap e60: 8 total, 8 up, 8 in 2024-04-27T02:04:21.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2984656555' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-27T02:04:21.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-27T02:04:21.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[26568]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:21.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[26568]: osdmap e60: 8 total, 8 up, 8 in 2024-04-27T02:04:21.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2984656555' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-27T02:04:21.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:21 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-27T02:04:22.129 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-27T02:04:22.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:22 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-27T02:04:22.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:22 smithi026 ceph-mon[26568]: osdmap e61: 8 total, 8 up, 8 in 2024-04-27T02:04:22.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:22 smithi026 ceph-mon[26568]: osdmap e62: 8 total, 8 up, 8 in 2024-04-27T02:04:22.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:22 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-27T02:04:22.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:22 smithi026 ceph-mon[21867]: osdmap e61: 8 total, 8 up, 8 in 2024-04-27T02:04:22.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:22 smithi026 ceph-mon[21867]: osdmap e62: 8 total, 8 up, 8 in 2024-04-27T02:04:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:22 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-27T02:04:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:22 smithi151 ceph-mon[22663]: osdmap e61: 8 total, 8 up, 8 in 2024-04-27T02:04:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:22 smithi151 ceph-mon[22663]: osdmap e62: 8 total, 8 up, 8 in 2024-04-27T02:04:22.777 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:22 smithi151 systemd[1]: Starting Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:04:23.056 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:22 smithi151 podman[38784]: 2024-04-27 02:04:22.699816343 +0000 UTC m=+0.017215426 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-27T02:04:23.056 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:22 smithi151 podman[38784]: 2024-04-27 02:04:22.807651037 +0000 UTC m=+0.125050164 container create 611ca410ba2628897feb74396608b8b0d3b582b2c308ad67fb512ae33e523c02 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, release=236.1648460182, description=Ceph Grafana Container, version=8.5, io.openshift.expose-services=, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-27T02:04:23.056 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 podman[38784]: 2024-04-27 02:04:23.076554962 +0000 UTC m=+0.393954047 container init 611ca410ba2628897feb74396608b8b0d3b582b2c308ad67fb512ae33e523c02 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, version=8.5) 2024-04-27T02:04:23.307 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 podman[38784]: 2024-04-27 02:04:23.079701806 +0000 UTC m=+0.397100891 container start 611ca410ba2628897feb74396608b8b0d3b582b2c308ad67fb512ae33e523c02 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=236.1648460182, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vcs-type=git, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, io.buildah.version=1.24.2, description=Ceph Grafana Container, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, distribution-scope=public, build-date=2022-03-28T10:36:18.413762) 2024-04-27T02:04:23.307 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-27T02:04:23.307 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-27T02:04:23.307 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-27T02:04:23.307 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-27T02:04:23.308 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-27T02:04:23.309 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-27T02:04:23.309 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="App mode production" logger=settings 2024-04-27T02:04:23.309 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-27T02:04:23.309 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-27T02:04:23.309 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-27T02:04:23.309 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-04-27T02:04:23.310 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-04-27T02:04:23.311 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-04-27T02:04:23.312 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-27T02:04:23.313 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-27T02:04:23.314 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-27T02:04:23.314 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-27T02:04:23.314 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-27T02:04:23.314 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-27T02:04:23.314 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-27T02:04:23.314 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-27T02:04:23.315 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 bash[38784]: 611ca410ba2628897feb74396608b8b0d3b582b2c308ad67fb512ae33e523c02 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 systemd[1]: Started Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-27T02:04:23.316 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-27T02:04:23.317 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-27T02:04:23.318 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-27T02:04:23.319 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-27T02:04:23.320 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-27T02:04:23.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-mon[22663]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:23.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:23.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:23.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:23.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:23.559 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-27T02:04:23.559 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-27T02:04:23.560 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-27T02:04:23.561 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-27T02:04:23.562 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-27T02:04:23.562 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-27T02:04:23.563 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-27T02:04:23.564 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-27T02:04:23.565 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-27T02:04:23.566 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-27T02:04:23.567 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-27T02:04:23.568 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-27T02:04:23.569 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-27T02:04:23.570 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-27T02:04:23.571 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-27T02:04:23.572 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-27T02:04:23.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-27T02:04:23.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-27T02:04:23.575 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-27T02:04:23.576 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-27T02:04:23.576 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-27T02:04:23.576 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-27T02:04:23.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[21867]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:23.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:23.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:23.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:23.715 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:23.715 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[26568]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:23.715 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:23.715 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:23.715 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:23.715 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:23 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:23.849 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-27T02:04:23.850 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-27T02:04:23.851 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-27T02:04:23.852 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-27T02:04:23.853 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=540.015531ms 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-27T02:04:23.854 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-27T02:04:23.855 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-27T02:04:23.855 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:04:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:04:23+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "e7b88545-1c1c-4e86-85de-73a9212039d1", 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "default", 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "api_name": "default", 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "is_master": "true", 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "endpoints": [], 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames": [], 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames_s3website": [], 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zone": "", 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "zones": [], 2024-04-27T02:04:23.883 INFO:teuthology.orchestra.run.smithi026.stdout: "placement_targets": [], 2024-04-27T02:04:23.884 INFO:teuthology.orchestra.run.smithi026.stdout: "default_placement": "", 2024-04-27T02:04:23.884 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "f69746c3-3a88-4f46-a5ee-0cbb789bc2e2", 2024-04-27T02:04:23.884 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_policy": { 2024-04-27T02:04:23.884 INFO:teuthology.orchestra.run.smithi026.stdout: "groups": [] 2024-04-27T02:04:23.884 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:23.884 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:04:24.466 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-27T02:04:25.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:25 smithi026 ceph-mon[21867]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:25.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:25 smithi026 ceph-mon[26568]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:25.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:25 smithi151 ceph-mon[22663]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:26.724 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "ef7cff2f-8f6b-4775-8d4e-c7ae081c911f", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "z", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "domain_root": "z.rgw.meta:root", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "control_pool": "z.rgw.control", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "log_pool": "z.rgw.log", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-27T02:04:26.725 INFO:teuthology.orchestra.run.smithi026.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "otp_pool": "z.rgw.otp", 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "system_key": { 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "access_key": "", 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "secret_key": "" 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "placement_pools": [ 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "key": "default-placement", 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "val": { 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "storage_classes": { 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "STANDARD": { 2024-04-27T02:04:26.726 INFO:teuthology.orchestra.run.smithi026.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: "index_type": 0 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "f69746c3-3a88-4f46-a5ee-0cbb789bc2e2", 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-27T02:04:26.727 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:04:27.346 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:04:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:26] "GET /metrics HTTP/1.1" 200 191478 "" "Prometheus/2.33.4" 2024-04-27T02:04:27.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:04:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:04:27.374 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-27T02:04:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:27 smithi026 ceph-mon[26568]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:27 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:27.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:27 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:27 smithi026 ceph-mon[21867]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:27 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:27.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:27 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:28.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:27 smithi151 ceph-mon[22663]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:28.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:27 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:28.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:27 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:29.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:29 smithi026 ceph-mon[21867]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 135 B/s rd, 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:29.803 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:29 smithi026 ceph-mon[26568]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 135 B/s rd, 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:30.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:29 smithi151 ceph-mon[22663]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 135 B/s rd, 1.3 KiB/s wr, 1 op/s 2024-04-27T02:04:31.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[21867]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-04-27T02:04:31.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:31.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:31.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[21867]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-27T02:04:31.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[21867]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-27T02:04:31.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[26568]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-04-27T02:04:31.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:31.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:31.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[26568]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-27T02:04:31.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:31 smithi026 ceph-mon[26568]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-27T02:04:31.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:31 smithi151 ceph-mon[22663]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-04-27T02:04:31.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:31 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:31.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:31 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:31.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:31 smithi151 ceph-mon[22663]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-27T02:04:31.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:31 smithi151 ceph-mon[22663]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-27T02:04:32.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:32 smithi151 ceph-mon[22663]: osdmap e63: 8 total, 8 up, 8 in 2024-04-27T02:04:32.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:32 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2398218478' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-27T02:04:32.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:32 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-27T02:04:32.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:32 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:32.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[26568]: osdmap e63: 8 total, 8 up, 8 in 2024-04-27T02:04:32.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2398218478' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-27T02:04:32.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-27T02:04:32.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:32.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[21867]: osdmap e63: 8 total, 8 up, 8 in 2024-04-27T02:04:32.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2398218478' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-27T02:04:32.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-27T02:04:32.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:32 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:33.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:33 smithi026 ceph-mon[21867]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-27T02:04:33.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:33 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-27T02:04:33.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:33 smithi026 ceph-mon[21867]: osdmap e64: 8 total, 8 up, 8 in 2024-04-27T02:04:33.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:33 smithi026 ceph-mon[26568]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-27T02:04:33.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:33 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-27T02:04:33.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:33 smithi026 ceph-mon[26568]: osdmap e64: 8 total, 8 up, 8 in 2024-04-27T02:04:33.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:33 smithi151 ceph-mon[22663]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-27T02:04:33.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:33 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-27T02:04:33.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:33 smithi151 ceph-mon[22663]: osdmap e64: 8 total, 8 up, 8 in 2024-04-27T02:04:34.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:33 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:04:34.109 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:33 smithi026 bash[46442]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[21867]: osdmap e65: 8 total, 8 up, 8 in 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2398218478' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[21867]: osdmap e66: 8 total, 8 up, 8 in 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[26568]: osdmap e65: 8 total, 8 up, 8 in 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2398218478' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-27T02:04:34.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-27T02:04:34.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-27T02:04:34.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-mon[26568]: osdmap e66: 8 total, 8 up, 8 in 2024-04-27T02:04:34.468 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[44742]: level=info ts=2024-04-27T02:04:34.130Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:04:34.468 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 podman[46455]: 2024-04-27 02:04:34.160936886 +0000 UTC m=+0.155055450 container died 50d1bfe425cce0f8231c696c5c16b5e78f24fd505d4b37e68b7e1e2ef4b9d1c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:34.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:34 smithi151 ceph-mon[22663]: osdmap e65: 8 total, 8 up, 8 in 2024-04-27T02:04:34.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:34 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2398218478' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-27T02:04:34.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:34 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-27T02:04:34.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:34 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-27T02:04:34.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:34 smithi151 ceph-mon[22663]: osdmap e66: 8 total, 8 up, 8 in 2024-04-27T02:04:34.794 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 podman[46455]: 2024-04-27 02:04:34.521329819 +0000 UTC m=+0.515448369 container cleanup 50d1bfe425cce0f8231c696c5c16b5e78f24fd505d4b37e68b7e1e2ef4b9d1c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:34.794 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 bash[46455]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a 2024-04-27T02:04:34.794 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 bash[46483]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-27T02:04:35.086 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 podman[46469]: 2024-04-27 02:04:34.81435271 +0000 UTC m=+0.651054989 container remove 50d1bfe425cce0f8231c696c5c16b5e78f24fd505d4b37e68b7e1e2ef4b9d1c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:35.086 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 podman[46469]: 2024-04-27 02:04:34.881760033 +0000 UTC m=+0.718462314 volume remove 00e8b7b115a02e94513bae6f6c7fd21c3c36dad9c5fc75d9bd252f4cc60c94fd 2024-04-27T02:04:35.086 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-27T02:04:35.087 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:04:35.087 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:34 smithi026 systemd[1]: Starting Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:04:35.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-mon[21867]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1023 B/s wr, 6 op/s 2024-04-27T02:04:35.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-mon[21867]: osdmap e67: 8 total, 8 up, 8 in 2024-04-27T02:04:35.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-mon[26568]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1023 B/s wr, 6 op/s 2024-04-27T02:04:35.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-mon[26568]: osdmap e67: 8 total, 8 up, 8 in 2024-04-27T02:04:35.359 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 podman[46564]: 2024-04-27 02:04:35.107377271 +0000 UTC m=+0.112618737 volume create 22774dc03a585c4b173fe1fe70815e7cf43a6565b54b969a48b52502ac67ba03 2024-04-27T02:04:35.359 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 podman[46564]: 2024-04-27 02:04:35.013283792 +0000 UTC m=+0.018525278 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-27T02:04:35.359 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 podman[46564]: 2024-04-27 02:04:35.216134026 +0000 UTC m=+0.221375492 container create 5cec00038ac8404a63763b4edba746d3a8f99567837eb85b8d1af52743b90dd5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:35.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:35 smithi151 ceph-mon[22663]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1023 B/s wr, 6 op/s 2024-04-27T02:04:35.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:35 smithi151 ceph-mon[22663]: osdmap e67: 8 total, 8 up, 8 in 2024-04-27T02:04:35.858 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 podman[46564]: 2024-04-27 02:04:35.52760899 +0000 UTC m=+0.532850457 container init 5cec00038ac8404a63763b4edba746d3a8f99567837eb85b8d1af52743b90dd5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:35.858 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 podman[46564]: 2024-04-27 02:04:35.530211065 +0000 UTC m=+0.535452533 container start 5cec00038ac8404a63763b4edba746d3a8f99567837eb85b8d1af52743b90dd5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:35.858 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.540Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.540Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.541Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.542Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.568Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.568Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.571Z caller=main.go:518 msg=Listening address=:9093 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:35.571Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 bash[46564]: 5cec00038ac8404a63763b4edba746d3a8f99567837eb85b8d1af52743b90dd5 2024-04-27T02:04:35.859 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:35 smithi026 systemd[1]: Started Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:04:36.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-27T02:04:36.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-27T02:04:36.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:36.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-mon[22663]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:04:36.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-mon[22663]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:04:36.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-27T02:04:36.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-mon[22663]: osdmap e68: 8 total, 8 up, 8 in 2024-04-27T02:04:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-27T02:04:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-27T02:04:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:04:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:04:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-27T02:04:36.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[21867]: osdmap e68: 8 total, 8 up, 8 in 2024-04-27T02:04:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-27T02:04:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-27T02:04:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[26568]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:04:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[26568]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:04:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-27T02:04:36.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-mon[26568]: osdmap e68: 8 total, 8 up, 8 in 2024-04-27T02:04:37.246 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:04:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:04:37.328 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:04:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:36] "GET /metrics HTTP/1.1" 200 194186 "" "Prometheus/2.33.4" 2024-04-27T02:04:37.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:37 smithi151 ceph-mon[22663]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-27T02:04:37.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:37 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-27T02:04:37.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:37 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-27T02:04:37.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:37 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-27T02:04:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:37 smithi151 ceph-mon[22663]: osdmap e69: 8 total, 8 up, 8 in 2024-04-27T02:04:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:37 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-27T02:04:37.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:37 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-27T02:04:37.608 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:37.543Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000316134s 2024-04-27T02:04:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[21867]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-27T02:04:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-27T02:04:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-27T02:04:37.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[21867]: osdmap e69: 8 total, 8 up, 8 in 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[26568]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[26568]: osdmap e69: 8 total, 8 up, 8 in 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3646666395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-27T02:04:37.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:37 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "45dd789f-315e-4375-b833-4c4c2b57b63c", 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "epoch": 1, 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "predecessor_uuid": "dc8ecf8f-b583-4bcd-be51-610f45b84b06", 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_status": [], 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "period_map": { 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "45dd789f-315e-4375-b833-4c4c2b57b63c", 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "zonegroups": [ 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "e7b88545-1c1c-4e86-85de-73a9212039d1", 2024-04-27T02:04:38.367 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "default", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "api_name": "default", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "is_master": "true", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "endpoints": [], 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames": [], 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames_s3website": [], 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zone": "ef7cff2f-8f6b-4775-8d4e-c7ae081c911f", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "zones": [ 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "ef7cff2f-8f6b-4775-8d4e-c7ae081c911f", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "z", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "endpoints": [], 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "log_meta": "false", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "log_data": "false", 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "bucket_index_max_shards": 11, 2024-04-27T02:04:38.368 INFO:teuthology.orchestra.run.smithi026.stdout: "read_only": "false", 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "tier_type": "", 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_from_all": "true", 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_from": [], 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "redirect_zone": "" 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "placement_targets": [ 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "default-placement", 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "tags": [], 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "storage_classes": [ 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "STANDARD" 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:04:38.369 INFO:teuthology.orchestra.run.smithi026.stdout: "default_placement": "default-placement", 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "f69746c3-3a88-4f46-a5ee-0cbb789bc2e2", 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_policy": { 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "groups": [] 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "short_zone_ids": [ 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "key": "ef7cff2f-8f6b-4775-8d4e-c7ae081c911f", 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "val": 3140115555 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zonegroup": "e7b88545-1c1c-4e86-85de-73a9212039d1", 2024-04-27T02:04:38.370 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zone": "ef7cff2f-8f6b-4775-8d4e-c7ae081c911f", 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "period_config": { 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "bucket_quota": { 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "check_on_raw": false, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size": -1, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size_kb": 0, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "max_objects": -1 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "user_quota": { 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "check_on_raw": false, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size": -1, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size_kb": 0, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "max_objects": -1 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:38.371 INFO:teuthology.orchestra.run.smithi026.stdout: "user_ratelimit": { 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_ops": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_ops": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_bytes": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_bytes": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "bucket_ratelimit": { 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_ops": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_ops": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_bytes": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_bytes": 0, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "anonymous_ratelimit": { 2024-04-27T02:04:38.372 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_ops": 0, 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_ops": 0, 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_bytes": 0, 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_bytes": 0, 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "f69746c3-3a88-4f46-a5ee-0cbb789bc2e2", 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_name": "r", 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_epoch": 2 2024-04-27T02:04:38.373 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:04:38.993 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:38 smithi151 systemd[1]: Stopping Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:04:38.994 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:38 smithi151 bash[39476]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:04:39.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-27T02:04:39.302 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.014Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-27T02:04:39.303 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.015Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-27T02:04:39.303 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.015Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-27T02:04:39.303 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[37789]: ts=2024-04-27T02:04:39.015Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-27T02:04:39.303 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 podman[39489]: 2024-04-27 02:04:39.024718664 +0000 UTC m=+0.113410674 container died a5b99015bed7e1bdd59bbb512a39760211790aeec56c0e50e9acfbe6efddc2f6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:39.303 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 podman[39489]: 2024-04-27 02:04:39.32243444 +0000 UTC m=+0.411126442 container cleanup a5b99015bed7e1bdd59bbb512a39760211790aeec56c0e50e9acfbe6efddc2f6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:39.303 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 bash[39489]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a 2024-04-27T02:04:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:39 smithi026 ceph-mon[26568]: pgmap v61: 129 pgs: 19 unknown, 110 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-27T02:04:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:39 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-27T02:04:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:39 smithi026 ceph-mon[26568]: osdmap e70: 8 total, 8 up, 8 in 2024-04-27T02:04:39.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:39 smithi026 ceph-mon[21867]: pgmap v61: 129 pgs: 19 unknown, 110 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-27T02:04:39.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:39 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-27T02:04:39.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:39 smithi026 ceph-mon[21867]: osdmap e70: 8 total, 8 up, 8 in 2024-04-27T02:04:39.562 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 bash[39515]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:04:39.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-mon[22663]: pgmap v61: 129 pgs: 19 unknown, 110 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-27T02:04:39.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-27T02:04:39.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:39 smithi151 ceph-mon[22663]: osdmap e70: 8 total, 8 up, 8 in 2024-04-27T02:04:39.849 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 podman[39503]: 2024-04-27 02:04:39.583295894 +0000 UTC m=+0.557499238 container remove a5b99015bed7e1bdd59bbb512a39760211790aeec56c0e50e9acfbe6efddc2f6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:39.849 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-27T02:04:39.849 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 systemd[1]: Stopped Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:04:39.849 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 systemd[1]: Starting Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:04:39.849 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 podman[39586]: 2024-04-27 02:04:39.733933104 +0000 UTC m=+0.019493853 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-27T02:04:40.115 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:39 smithi151 podman[39586]: 2024-04-27 02:04:39.876013367 +0000 UTC m=+0.161574105 container create 6c19ce81f41e80181eeadcc8761eca75b2af45ed28e142fb31eef540669f0e3b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:40.115 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 podman[39586]: 2024-04-27 02:04:40.069622754 +0000 UTC m=+0.355183497 container init 6c19ce81f41e80181eeadcc8761eca75b2af45ed28e142fb31eef540669f0e3b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:40.115 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 podman[39586]: 2024-04-27 02:04:40.07207556 +0000 UTC m=+0.357636299 container start 6c19ce81f41e80181eeadcc8761eca75b2af45ed28e142fb31eef540669f0e3b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.094Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.094Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.094Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.095Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi151 (none))" 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.095Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.095Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.096Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.097Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.098Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.099Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.099Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.316µs 2024-04-27T02:04:40.116 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.099Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-27T02:04:40.117 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 bash[39586]: 6c19ce81f41e80181eeadcc8761eca75b2af45ed28e142fb31eef540669f0e3b 2024-04-27T02:04:40.599 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 systemd[1]: Started Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:04:40.599 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.307Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-27T02:04:40.599 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.307Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-27T02:04:40.599 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.307Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.225µs wal_replay_duration=207.789069ms total_replay_duration=207.826442ms 2024-04-27T02:04:40.599 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.309Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-27T02:04:40.599 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.309Z caller=main.go:947 level=info msg="TSDB started" 2024-04-27T02:04:40.599 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.309Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-27T02:04:40.600 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.325Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.55036ms db_storage=695ns remote_storage=1.341µs web_handler=512ns query_engine=660ns scrape=494.481µs scrape_sd=23.016µs notify=25.915µs notify_sd=11.23µs rules=14.749162ms 2024-04-27T02:04:40.600 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:04:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:04:40.325Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-27T02:04:41.266 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled rgw.foo update... 2024-04-27T02:04:41.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 436 B/s wr, 1 op/s 2024-04-27T02:04:41.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:04:41.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:04:41.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-27T02:04:41.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-27T02:04:41.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:41.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 436 B/s wr, 1 op/s 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2024-04-27T02:04:41.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:41.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:41 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:41.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 436 B/s wr, 1 op/s 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:04:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:04:41.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2024-04-27T02:04:41.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2024-04-27T02:04:41.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:41.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:41.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:41.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:41 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:42.036 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph osd pool create foo' 2024-04-27T02:04:42.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:42 smithi151 ceph-mon[22663]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:04:42.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:42 smithi151 ceph-mon[22663]: Saving service rgw.foo spec with placement count:2 2024-04-27T02:04:42.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:42 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:42.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:42 smithi026 ceph-mon[26568]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:04:42.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:42 smithi026 ceph-mon[26568]: Saving service rgw.foo spec with placement count:2 2024-04-27T02:04:42.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:42 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:42.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:42 smithi026 ceph-mon[21867]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:04:42.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:42 smithi026 ceph-mon[21867]: Saving service rgw.foo spec with placement count:2 2024-04-27T02:04:42.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:42 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:43.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:43 smithi026 ceph-mon[21867]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-27T02:04:43.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:43 smithi026 ceph-mon[26568]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-27T02:04:43.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:43 smithi151 ceph-mon[22663]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-27T02:04:44.788 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'foo' created 2024-04-27T02:04:45.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:44 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1198151888' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-27T02:04:45.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:44 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-27T02:04:45.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:44 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1198151888' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-27T02:04:45.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:44 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-27T02:04:45.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:44 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1198151888' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-27T02:04:45.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:44 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-27T02:04:45.690 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'rbd pool init foo' 2024-04-27T02:04:45.778 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:04:45 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:04:45.544Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002026874s 2024-04-27T02:04:46.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:45 smithi026 ceph-mon[26568]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-27T02:04:46.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:45 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-27T02:04:46.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:45 smithi026 ceph-mon[26568]: osdmap e71: 8 total, 8 up, 8 in 2024-04-27T02:04:46.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:45 smithi026 ceph-mon[21867]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-27T02:04:46.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:45 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-27T02:04:46.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:45 smithi026 ceph-mon[21867]: osdmap e71: 8 total, 8 up, 8 in 2024-04-27T02:04:46.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:45 smithi151 ceph-mon[22663]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-27T02:04:46.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:45 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-27T02:04:46.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:45 smithi151 ceph-mon[22663]: osdmap e71: 8 total, 8 up, 8 in 2024-04-27T02:04:46.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: osdmap e72: 8 total, 8 up, 8 in 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:46.928 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: osdmap e72: 8 total, 8 up, 8 in 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:46.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:47.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:04:47.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: osdmap e72: 8 total, 8 up, 8 in 2024-04-27T02:04:47.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:47.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:47.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:46 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:47.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:04:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:46] "GET /metrics HTTP/1.1" 200 194186 "" "Prometheus/2.33.4" 2024-04-27T02:04:47.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[26568]: pgmap v68: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 931 B/s wr, 2 op/s 2024-04-27T02:04:47.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[26568]: Saving service rgw.foo spec with placement count:2 2024-04-27T02:04:47.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[26568]: Deploying daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:04:47.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[26568]: osdmap e73: 8 total, 8 up, 8 in 2024-04-27T02:04:47.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[21867]: pgmap v68: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 931 B/s wr, 2 op/s 2024-04-27T02:04:47.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[21867]: Saving service rgw.foo spec with placement count:2 2024-04-27T02:04:47.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[21867]: Deploying daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:04:47.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:47 smithi026 ceph-mon[21867]: osdmap e73: 8 total, 8 up, 8 in 2024-04-27T02:04:48.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:47 smithi151 ceph-mon[22663]: pgmap v68: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 931 B/s wr, 2 op/s 2024-04-27T02:04:48.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:47 smithi151 ceph-mon[22663]: Saving service rgw.foo spec with placement count:2 2024-04-27T02:04:48.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:47 smithi151 ceph-mon[22663]: Deploying daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:04:48.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:47 smithi151 ceph-mon[22663]: osdmap e73: 8 total, 8 up, 8 in 2024-04-27T02:04:49.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:48 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2055849231' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-27T02:04:49.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:48 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2055849231' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-27T02:04:49.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:48 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2055849231' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-27T02:04:50.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:49 smithi151 ceph-mon[22663]: pgmap v70: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-27T02:04:50.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:49 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2055849231' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-27T02:04:50.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:49 smithi151 ceph-mon[22663]: osdmap e74: 8 total, 8 up, 8 in 2024-04-27T02:04:50.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:49 smithi026 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-27T02:04:50.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:49 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2055849231' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-27T02:04:50.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:49 smithi026 ceph-mon[21867]: osdmap e74: 8 total, 8 up, 8 in 2024-04-27T02:04:50.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:49 smithi026 ceph-mon[26568]: pgmap v70: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-27T02:04:50.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:49 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2055849231' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-27T02:04:50.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:49 smithi026 ceph-mon[26568]: osdmap e74: 8 total, 8 up, 8 in 2024-04-27T02:04:51.033 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:50 smithi151 ceph-mon[22663]: osdmap e75: 8 total, 8 up, 8 in 2024-04-27T02:04:51.033 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:50 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:51.033 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:50 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:04:51.034 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:50 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-27T02:04:51.034 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:50 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:51.034 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:50 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:51.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[21867]: osdmap e75: 8 total, 8 up, 8 in 2024-04-27T02:04:51.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:51.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:04:51.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[26568]: osdmap e75: 8 total, 8 up, 8 in 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:51.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:50 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:51.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-27T02:04:52.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:51 smithi026 ceph-mon[26568]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:52.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:51 smithi026 ceph-mon[26568]: Deploying daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:04:52.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:51 smithi026 ceph-mon[26568]: osdmap e76: 8 total, 8 up, 8 in 2024-04-27T02:04:52.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:51 smithi026 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:52.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:51 smithi026 ceph-mon[21867]: Deploying daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:04:52.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:51 smithi026 ceph-mon[21867]: osdmap e76: 8 total, 8 up, 8 in 2024-04-27T02:04:52.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:51 smithi151 ceph-mon[22663]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:52.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:51 smithi151 ceph-mon[22663]: Deploying daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:04:52.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:51 smithi151 ceph-mon[22663]: osdmap e76: 8 total, 8 up, 8 in 2024-04-27T02:04:53.638 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled iscsi.foo update... 2024-04-27T02:04:53.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[26568]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:53.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:53 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:54.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:53 smithi151 ceph-mon[22663]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:04:54.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:53 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:54.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:53 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:04:54.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:53 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:54.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:53 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:04:54.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:53 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:54.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:54 smithi151 ceph-mon[22663]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:04:54.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:54 smithi151 ceph-mon[22663]: Saving service iscsi.foo spec with placement count:1 2024-04-27T02:04:55.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:54 smithi026 ceph-mon[21867]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:04:55.025 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:54 smithi026 ceph-mon[21867]: Saving service iscsi.foo spec with placement count:1 2024-04-27T02:04:55.025 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:54 smithi026 ceph-mon[26568]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:04:55.025 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:54 smithi026 ceph-mon[26568]: Saving service iscsi.foo spec with placement count:1 2024-04-27T02:04:55.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 180' 2024-04-27T02:04:56.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:55 smithi026 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 6.0 KiB/s wr, 419 op/s 2024-04-27T02:04:56.084 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:55 smithi026 ceph-mon[26568]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 6.0 KiB/s wr, 419 op/s 2024-04-27T02:04:56.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:55 smithi151 ceph-mon[22663]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 6.0 KiB/s wr, 419 op/s 2024-04-27T02:04:57.108 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:04:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:56] "GET /metrics HTTP/1.1" 200 197860 "" "Prometheus/2.33.4" 2024-04-27T02:04:57.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:04:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:04:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:04:57.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:57 smithi151 ceph-mon[22663]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 188 KiB/s rd, 5.0 KiB/s wr, 348 op/s 2024-04-27T02:04:57.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:57 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:57.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:57 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:57.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:57 smithi026 ceph-mon[26568]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 188 KiB/s rd, 5.0 KiB/s wr, 348 op/s 2024-04-27T02:04:57.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:57 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:57.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:57 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:57.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:57 smithi026 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 188 KiB/s rd, 5.0 KiB/s wr, 348 op/s 2024-04-27T02:04:57.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:57 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:57.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:57 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: osdmap e77: 8 total, 8 up, 8 in 2024-04-27T02:04:59.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:04:59.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-27T02:04:59.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:58 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: osdmap e77: 8 total, 8 up, 8 in 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-27T02:04:59.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: osdmap e77: 8 total, 8 up, 8 in 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-27T02:04:59.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:58 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:05:00.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[26568]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 225 KiB/s rd, 4.5 KiB/s wr, 400 op/s 2024-04-27T02:05:00.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[26568]: Checking dashboard <-> RGW credentials 2024-04-27T02:05:00.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[26568]: Checking pool "foo" exists for service iscsi.foo 2024-04-27T02:05:00.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[26568]: Deploying daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:05:00.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 225 KiB/s rd, 4.5 KiB/s wr, 400 op/s 2024-04-27T02:05:00.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[21867]: Checking dashboard <-> RGW credentials 2024-04-27T02:05:00.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[21867]: Checking pool "foo" exists for service iscsi.foo 2024-04-27T02:05:00.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:04:59 smithi026 ceph-mon[21867]: Deploying daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:05:00.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:59 smithi151 ceph-mon[22663]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 225 KiB/s rd, 4.5 KiB/s wr, 400 op/s 2024-04-27T02:05:00.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:59 smithi151 ceph-mon[22663]: Checking dashboard <-> RGW credentials 2024-04-27T02:05:00.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:59 smithi151 ceph-mon[22663]: Checking pool "foo" exists for service iscsi.foo 2024-04-27T02:05:00.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:04:59 smithi151 ceph-mon[22663]: Deploying daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:05:01.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:00 smithi026 ceph-mon[21867]: mgrmap e21: y(active, since 118s), standbys: x 2024-04-27T02:05:01.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:00 smithi026 ceph-mon[26568]: mgrmap e21: y(active, since 118s), standbys: x 2024-04-27T02:05:01.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:00 smithi151 ceph-mon[22663]: mgrmap e21: y(active, since 118s), standbys: x 2024-04-27T02:05:02.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:01 smithi151 ceph-mon[22663]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 196 KiB/s rd, 3.9 KiB/s wr, 348 op/s 2024-04-27T02:05:02.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:01 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:02.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:01 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:05:02.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:01 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:05:02.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:01 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 196 KiB/s rd, 3.9 KiB/s wr, 348 op/s 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[26568]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 196 KiB/s rd, 3.9 KiB/s wr, 348 op/s 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:02.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:05:02.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:05:02.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:01 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:05:02.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-27T02:05:02.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-27T02:05:02.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-27T02:05:02.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-04-27T02:05:02.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3781788841' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2055201164' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2363584941"}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:05:02.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:02.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3781788841' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:05:02.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:02 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2055201164' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2363584941"}]: dispatch 2024-04-27T02:05:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-27T02:05:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-27T02:05:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-27T02:05:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-04-27T02:05:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:05:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:05:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3781788841' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:05:03.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:02 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2055201164' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2363584941"}]: dispatch 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 3.6 KiB/s wr, 320 op/s 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2055201164' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2363584941"}]': finished 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: osdmap e78: 8 total, 8 up, 8 in 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3408782004' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]: dispatch 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]: dispatch 2024-04-27T02:05:04.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 3.6 KiB/s wr, 320 op/s 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2055201164' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2363584941"}]': finished 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: osdmap e78: 8 total, 8 up, 8 in 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3408782004' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]: dispatch 2024-04-27T02:05:04.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:03 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]: dispatch 2024-04-27T02:05:04.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 3.6 KiB/s wr, 320 op/s 2024-04-27T02:05:04.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-27T02:05:04.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-27T02:05:04.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-27T02:05:04.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-04-27T02:05:04.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2055201164' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2363584941"}]': finished 2024-04-27T02:05:04.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: osdmap e78: 8 total, 8 up, 8 in 2024-04-27T02:05:04.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3408782004' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]: dispatch 2024-04-27T02:05:04.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:03 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]: dispatch 2024-04-27T02:05:05.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]': finished 2024-04-27T02:05:05.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[21867]: osdmap e79: 8 total, 8 up, 8 in 2024-04-27T02:05:05.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3547237782' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]: dispatch 2024-04-27T02:05:05.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]: dispatch 2024-04-27T02:05:05.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]': finished 2024-04-27T02:05:05.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[26568]: osdmap e79: 8 total, 8 up, 8 in 2024-04-27T02:05:05.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3547237782' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]: dispatch 2024-04-27T02:05:05.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:04 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]: dispatch 2024-04-27T02:05:05.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:04 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1966443158"}]': finished 2024-04-27T02:05:05.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:04 smithi151 ceph-mon[22663]: osdmap e79: 8 total, 8 up, 8 in 2024-04-27T02:05:05.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:04 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3547237782' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]: dispatch 2024-04-27T02:05:05.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:04 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]: dispatch 2024-04-27T02:05:06.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 492 B/s wr, 131 op/s 2024-04-27T02:05:06.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]': finished 2024-04-27T02:05:06.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: osdmap e80: 8 total, 8 up, 8 in 2024-04-27T02:05:06.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2407810566' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1866163037"}]: dispatch 2024-04-27T02:05:06.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:05:06.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:05 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 492 B/s wr, 131 op/s 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]': finished 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: osdmap e80: 8 total, 8 up, 8 in 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2407810566' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1866163037"}]: dispatch 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 492 B/s wr, 131 op/s 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/930416246"}]': finished 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: osdmap e80: 8 total, 8 up, 8 in 2024-04-27T02:05:06.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2407810566' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1866163037"}]: dispatch 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:05:06.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:05 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:05:07.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:05:07.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: Adding iSCSI gateway http://:@172.21.15.26:5000 to Dashboard 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: Checking dashboard <-> RGW credentials 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2407810566' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1866163037"}]': finished 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: osdmap e81: 8 total, 8 up, 8 in 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1972644663' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]: dispatch 2024-04-27T02:05:07.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:06 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]: dispatch 2024-04-27T02:05:07.108 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:06] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-04-27T02:05:07.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:05:07.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: Adding iSCSI gateway http://:@172.21.15.26:5000 to Dashboard 2024-04-27T02:05:07.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:05:07.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-27T02:05:07.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: Checking dashboard <-> RGW credentials 2024-04-27T02:05:07.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2407810566' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1866163037"}]': finished 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: osdmap e81: 8 total, 8 up, 8 in 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1972644663' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]: dispatch 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]: dispatch 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: Adding iSCSI gateway http://:@172.21.15.26:5000 to Dashboard 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: Checking dashboard <-> RGW credentials 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2407810566' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1866163037"}]': finished 2024-04-27T02:05:07.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: osdmap e81: 8 total, 8 up, 8 in 2024-04-27T02:05:07.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1972644663' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]: dispatch 2024-04-27T02:05:07.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:06 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]: dispatch 2024-04-27T02:05:08.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:07 smithi151 ceph-mon[22663]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 767 B/s wr, 31 op/s 2024-04-27T02:05:08.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:07 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]': finished 2024-04-27T02:05:08.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:07 smithi151 ceph-mon[22663]: osdmap e82: 8 total, 8 up, 8 in 2024-04-27T02:05:08.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:07 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2768414261' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]: dispatch 2024-04-27T02:05:08.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:07 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]: dispatch 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[21867]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 767 B/s wr, 31 op/s 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]': finished 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[21867]: osdmap e82: 8 total, 8 up, 8 in 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2768414261' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]: dispatch 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]: dispatch 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[26568]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 767 B/s wr, 31 op/s 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/124039062"}]': finished 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[26568]: osdmap e82: 8 total, 8 up, 8 in 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2768414261' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]: dispatch 2024-04-27T02:05:08.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:07 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]: dispatch 2024-04-27T02:05:09.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:08 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]': finished 2024-04-27T02:05:09.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:08 smithi151 ceph-mon[22663]: osdmap e83: 8 total, 8 up, 8 in 2024-04-27T02:05:09.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:08 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1948822146' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]: dispatch 2024-04-27T02:05:09.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:08 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]: dispatch 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]': finished 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[21867]: osdmap e83: 8 total, 8 up, 8 in 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1948822146' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]: dispatch 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]: dispatch 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1966443158"}]': finished 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[26568]: osdmap e83: 8 total, 8 up, 8 in 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1948822146' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]: dispatch 2024-04-27T02:05:09.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:08 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]: dispatch 2024-04-27T02:05:10.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:09 smithi151 ceph-mon[22663]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-27T02:05:10.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:09 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]': finished 2024-04-27T02:05:10.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:09 smithi151 ceph-mon[22663]: osdmap e84: 8 total, 8 up, 8 in 2024-04-27T02:05:10.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:09 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/715546664' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]: dispatch 2024-04-27T02:05:10.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:09 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]: dispatch 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]': finished 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[21867]: osdmap e84: 8 total, 8 up, 8 in 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/715546664' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]: dispatch 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]: dispatch 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[26568]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/384921090"}]': finished 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[26568]: osdmap e84: 8 total, 8 up, 8 in 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/715546664' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]: dispatch 2024-04-27T02:05:10.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:09 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]: dispatch 2024-04-27T02:05:11.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]': finished 2024-04-27T02:05:11.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[21867]: osdmap e85: 8 total, 8 up, 8 in 2024-04-27T02:05:11.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3445938035' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]: dispatch 2024-04-27T02:05:11.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]: dispatch 2024-04-27T02:05:11.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]': finished 2024-04-27T02:05:11.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[26568]: osdmap e85: 8 total, 8 up, 8 in 2024-04-27T02:05:11.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3445938035' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]: dispatch 2024-04-27T02:05:11.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:10 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]: dispatch 2024-04-27T02:05:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:10 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/358798680"}]': finished 2024-04-27T02:05:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:10 smithi151 ceph-mon[22663]: osdmap e85: 8 total, 8 up, 8 in 2024-04-27T02:05:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:10 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3445938035' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]: dispatch 2024-04-27T02:05:11.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:10 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]: dispatch 2024-04-27T02:05:12.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-27T02:05:12.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]': finished 2024-04-27T02:05:12.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[21867]: osdmap e86: 8 total, 8 up, 8 in 2024-04-27T02:05:12.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2430846066' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2505340537"}]: dispatch 2024-04-27T02:05:12.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[26568]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-27T02:05:12.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]': finished 2024-04-27T02:05:12.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[26568]: osdmap e86: 8 total, 8 up, 8 in 2024-04-27T02:05:12.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:11 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2430846066' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2505340537"}]: dispatch 2024-04-27T02:05:12.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:11 smithi151 ceph-mon[22663]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-27T02:05:12.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:11 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/102542096"}]': finished 2024-04-27T02:05:12.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:11 smithi151 ceph-mon[22663]: osdmap e86: 8 total, 8 up, 8 in 2024-04-27T02:05:12.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:11 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2430846066' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2505340537"}]: dispatch 2024-04-27T02:05:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:12 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:12 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2430846066' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2505340537"}]': finished 2024-04-27T02:05:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:12 smithi151 ceph-mon[22663]: osdmap e87: 8 total, 8 up, 8 in 2024-04-27T02:05:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:12 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2231831898' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]: dispatch 2024-04-27T02:05:13.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:12 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]: dispatch 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2430846066' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2505340537"}]': finished 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[21867]: osdmap e87: 8 total, 8 up, 8 in 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2231831898' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]: dispatch 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]: dispatch 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2430846066' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2505340537"}]': finished 2024-04-27T02:05:13.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[26568]: osdmap e87: 8 total, 8 up, 8 in 2024-04-27T02:05:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2231831898' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]: dispatch 2024-04-27T02:05:13.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:12 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]: dispatch 2024-04-27T02:05:13.858 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:05:13.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:05:13.858 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:05:13.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:05:14.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:13 smithi151 ceph-mon[22663]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:05:14.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:13 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]': finished 2024-04-27T02:05:14.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:13 smithi151 ceph-mon[22663]: osdmap e88: 8 total, 8 up, 8 in 2024-04-27T02:05:14.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:13 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4020424495' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1573366204"}]: dispatch 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]': finished 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[21867]: osdmap e88: 8 total, 8 up, 8 in 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4020424495' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1573366204"}]: dispatch 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[26568]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/518236597"}]': finished 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[26568]: osdmap e88: 8 total, 8 up, 8 in 2024-04-27T02:05:14.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:13 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4020424495' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1573366204"}]: dispatch 2024-04-27T02:05:15.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:14 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4020424495' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1573366204"}]': finished 2024-04-27T02:05:15.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:14 smithi151 ceph-mon[22663]: osdmap e89: 8 total, 8 up, 8 in 2024-04-27T02:05:15.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:14 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/269273943' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1573366204"}]: dispatch 2024-04-27T02:05:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:14 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4020424495' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1573366204"}]': finished 2024-04-27T02:05:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:14 smithi026 ceph-mon[21867]: osdmap e89: 8 total, 8 up, 8 in 2024-04-27T02:05:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:14 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/269273943' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1573366204"}]: dispatch 2024-04-27T02:05:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:14 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4020424495' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1573366204"}]': finished 2024-04-27T02:05:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:14 smithi026 ceph-mon[26568]: osdmap e89: 8 total, 8 up, 8 in 2024-04-27T02:05:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:14 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/269273943' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1573366204"}]: dispatch 2024-04-27T02:05:16.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:15 smithi151 ceph-mon[22663]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:16.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:15 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/269273943' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1573366204"}]': finished 2024-04-27T02:05:16.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:15 smithi151 ceph-mon[22663]: osdmap e90: 8 total, 8 up, 8 in 2024-04-27T02:05:16.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:15 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3148466995' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/518236597"}]: dispatch 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/269273943' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1573366204"}]': finished 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[21867]: osdmap e90: 8 total, 8 up, 8 in 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3148466995' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/518236597"}]: dispatch 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[26568]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/269273943' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1573366204"}]': finished 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[26568]: osdmap e90: 8 total, 8 up, 8 in 2024-04-27T02:05:16.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:15 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3148466995' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/518236597"}]: dispatch 2024-04-27T02:05:17.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:05:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:05:17.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:16 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3148466995' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/518236597"}]': finished 2024-04-27T02:05:17.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:16 smithi151 ceph-mon[22663]: osdmap e91: 8 total, 8 up, 8 in 2024-04-27T02:05:17.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:16 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2530183927' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]: dispatch 2024-04-27T02:05:17.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:16 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]: dispatch 2024-04-27T02:05:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3148466995' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/518236597"}]': finished 2024-04-27T02:05:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[21867]: osdmap e91: 8 total, 8 up, 8 in 2024-04-27T02:05:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2530183927' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]: dispatch 2024-04-27T02:05:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]: dispatch 2024-04-27T02:05:17.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3148466995' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/518236597"}]': finished 2024-04-27T02:05:17.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[26568]: osdmap e91: 8 total, 8 up, 8 in 2024-04-27T02:05:17.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2530183927' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]: dispatch 2024-04-27T02:05:17.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]: dispatch 2024-04-27T02:05:17.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:05:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:16] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-04-27T02:05:18.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:17 smithi151 ceph-mon[22663]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:18.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:17 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]': finished 2024-04-27T02:05:18.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:17 smithi151 ceph-mon[22663]: osdmap e92: 8 total, 8 up, 8 in 2024-04-27T02:05:18.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:17 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1837971674' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]: dispatch 2024-04-27T02:05:18.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:17 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]: dispatch 2024-04-27T02:05:18.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[26568]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:18.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]': finished 2024-04-27T02:05:18.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[26568]: osdmap e92: 8 total, 8 up, 8 in 2024-04-27T02:05:18.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1837971674' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]: dispatch 2024-04-27T02:05:18.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]: dispatch 2024-04-27T02:05:18.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:18.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4281274133"}]': finished 2024-04-27T02:05:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[21867]: osdmap e92: 8 total, 8 up, 8 in 2024-04-27T02:05:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1837971674' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]: dispatch 2024-04-27T02:05:18.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:17 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]: dispatch 2024-04-27T02:05:19.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:18 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]': finished 2024-04-27T02:05:19.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:18 smithi026 ceph-mon[21867]: osdmap e93: 8 total, 8 up, 8 in 2024-04-27T02:05:19.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:18 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]': finished 2024-04-27T02:05:19.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:18 smithi026 ceph-mon[26568]: osdmap e93: 8 total, 8 up, 8 in 2024-04-27T02:05:19.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:18 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1738905563"}]': finished 2024-04-27T02:05:19.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:18 smithi151 ceph-mon[22663]: osdmap e93: 8 total, 8 up, 8 in 2024-04-27T02:05:20.276 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:19 smithi026 ceph-mon[21867]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:20.276 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:19 smithi026 ceph-mon[26568]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:20.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:19 smithi151 ceph-mon[22663]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:22.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:21 smithi026 ceph-mon[21867]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-04-27T02:05:22.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:21 smithi026 ceph-mon[26568]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-04-27T02:05:22.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:21 smithi151 ceph-mon[22663]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-04-27T02:05:23.167 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:22 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:23.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:22 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:23.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:22 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:23.858 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:05:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:05:23.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:05:24.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:23 smithi151 ceph-mon[22663]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-04-27T02:05:24.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:23 smithi026 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-04-27T02:05:24.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:23 smithi026 ceph-mon[26568]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-04-27T02:05:26.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:25 smithi151 ceph-mon[22663]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:26.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:25 smithi026 ceph-mon[21867]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:26.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:25 smithi026 ceph-mon[26568]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:27.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:05:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:05:27.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:05:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:26] "GET /metrics HTTP/1.1" 200 208130 "" "Prometheus/2.33.4" 2024-04-27T02:05:28.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:27 smithi151 ceph-mon[22663]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:05:28.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:27 smithi026 ceph-mon[21867]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:05:28.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:27 smithi026 ceph-mon[26568]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:05:30.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:29 smithi151 ceph-mon[22663]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-27T02:05:30.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:29 smithi026 ceph-mon[21867]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-27T02:05:30.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:29 smithi026 ceph-mon[26568]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-27T02:05:32.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:31 smithi151 ceph-mon[22663]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:32.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:31 smithi026 ceph-mon[21867]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:32.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:31 smithi026 ceph-mon[26568]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:33.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:32 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:33.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:32 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:33.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:32 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:34.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:33 smithi151 ceph-mon[22663]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:34.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:33 smithi026 ceph-mon[21867]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:34.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:33 smithi026 ceph-mon[26568]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:36.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:35 smithi151 ceph-mon[22663]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:36.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:35 smithi026 ceph-mon[21867]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:36.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:35 smithi026 ceph-mon[26568]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:37.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:05:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:05:37.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:05:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:36] "GET /metrics HTTP/1.1" 200 208084 "" "Prometheus/2.33.4" 2024-04-27T02:05:38.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:37 smithi151 ceph-mon[22663]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:38.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:37 smithi026 ceph-mon[21867]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:38.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:37 smithi026 ceph-mon[26568]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:40.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:39 smithi151 ceph-mon[22663]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:40.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:39 smithi026 ceph-mon[21867]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:40.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:39 smithi026 ceph-mon[26568]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:42.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:41 smithi151 ceph-mon[22663]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:42.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:41 smithi026 ceph-mon[21867]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:42.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:41 smithi026 ceph-mon[26568]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:42 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:43.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:42 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:43.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:42 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:44.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:43 smithi151 ceph-mon[22663]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:44.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:43 smithi026 ceph-mon[21867]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:44.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:43 smithi026 ceph-mon[26568]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:46.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:45 smithi151 ceph-mon[22663]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:46.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:45 smithi026 ceph-mon[21867]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:46.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:45 smithi026 ceph-mon[26568]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:47.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:05:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:05:47.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:05:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:46] "GET /metrics HTTP/1.1" 200 208084 "" "Prometheus/2.33.4" 2024-04-27T02:05:48.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:47 smithi151 ceph-mon[22663]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:48.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:47 smithi026 ceph-mon[21867]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:47 smithi026 ceph-mon[26568]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:50.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:49 smithi151 ceph-mon[22663]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:50.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:49 smithi026 ceph-mon[21867]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:50.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:49 smithi026 ceph-mon[26568]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:52.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:51 smithi151 ceph-mon[22663]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:52.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:51 smithi026 ceph-mon[21867]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:52.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:51 smithi026 ceph-mon[26568]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:53.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:52 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:53.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:52 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:53.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:52 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:05:54.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:53 smithi151 ceph-mon[22663]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:54.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:53 smithi026 ceph-mon[21867]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:54.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:53 smithi026 ceph-mon[26568]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:56.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:55 smithi151 ceph-mon[22663]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:56.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:55 smithi026 ceph-mon[21867]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:55 smithi026 ceph-mon[26568]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:05:57.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:05:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:05:57.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:05:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:05:56] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-04-27T02:05:58.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:57 smithi151 ceph-mon[22663]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:58.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:57 smithi026 ceph-mon[21867]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:05:58.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:57 smithi026 ceph-mon[26568]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:00.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:05:59 smithi151 ceph-mon[22663]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:00.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:05:59 smithi026 ceph-mon[21867]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:00.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:05:59 smithi026 ceph-mon[26568]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:02.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:01 smithi151 ceph-mon[22663]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:02.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:01 smithi026 ceph-mon[21867]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:02.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:01 smithi026 ceph-mon[26568]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:03 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:06:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:06:03.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:03 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:03.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:06:03.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:06:03.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:03 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:03.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:06:03.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:06:04.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:04 smithi151 ceph-mon[22663]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:04.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:04 smithi026 ceph-mon[21867]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:04.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:04 smithi026 ceph-mon[26568]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:06.015 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:06 smithi151 ceph-mon[22663]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:06.015 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:06:06.015 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:06:06.015 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:06:06.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[21867]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:06.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:06:06.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:06:06.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:06:06.031 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[26568]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:06.031 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:06:06.031 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:06:06.031 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:06:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:07 smithi151 ceph-mon[22663]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:07 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:06:07.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:06:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:06:07.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:07 smithi026 ceph-mon[21867]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:07.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:07 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:06:07.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:07 smithi026 ceph-mon[26568]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:07.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:07 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:06:07.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:06:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:06] "GET /metrics HTTP/1.1" 200 208024 "" "Prometheus/2.33.4" 2024-04-27T02:06:10.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:09 smithi151 ceph-mon[22663]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:10.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:09 smithi026 ceph-mon[21867]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:10.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:09 smithi026 ceph-mon[26568]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:12.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:11 smithi151 ceph-mon[22663]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:12.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:11 smithi026 ceph-mon[21867]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:12.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:11 smithi026 ceph-mon[26568]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:13.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:12 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:13.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:12 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:13.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:12 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:14.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:13 smithi151 ceph-mon[22663]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:14.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:13 smithi026 ceph-mon[21867]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:14.151 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:13 smithi026 ceph-mon[26568]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:16.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:15 smithi151 ceph-mon[22663]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:15 smithi026 ceph-mon[21867]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:16.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:15 smithi026 ceph-mon[26568]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:17.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:06:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:06:17.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:06:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:16] "GET /metrics HTTP/1.1" 200 208024 "" "Prometheus/2.33.4" 2024-04-27T02:06:18.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:17 smithi151 ceph-mon[22663]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:18.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:17 smithi026 ceph-mon[21867]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:18.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:17 smithi026 ceph-mon[26568]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:20.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:19 smithi151 ceph-mon[22663]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:20.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:19 smithi026 ceph-mon[21867]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:20.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:19 smithi026 ceph-mon[26568]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:22.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:21 smithi151 ceph-mon[22663]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:22.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:21 smithi026 ceph-mon[21867]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:22.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:21 smithi026 ceph-mon[26568]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:23.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:22 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:23.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:22 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:23.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:22 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:24.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:23 smithi151 ceph-mon[22663]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:24.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:23 smithi026 ceph-mon[21867]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:24.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:23 smithi026 ceph-mon[26568]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:26.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:25 smithi151 ceph-mon[22663]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:26.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:25 smithi026 ceph-mon[21867]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:26.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:25 smithi026 ceph-mon[26568]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:27.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:06:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:06:27.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:06:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:26] "GET /metrics HTTP/1.1" 200 208034 "" "Prometheus/2.33.4" 2024-04-27T02:06:28.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:27 smithi151 ceph-mon[22663]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:28.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:27 smithi026 ceph-mon[21867]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:28.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:27 smithi026 ceph-mon[26568]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:30.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:29 smithi151 ceph-mon[22663]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:30.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:29 smithi026 ceph-mon[21867]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:30.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:29 smithi026 ceph-mon[26568]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:32.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:31 smithi151 ceph-mon[22663]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:32.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:31 smithi026 ceph-mon[21867]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:32.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:31 smithi026 ceph-mon[26568]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:33.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:32 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:33.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:32 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:33.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:32 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:34.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:33 smithi151 ceph-mon[22663]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:34.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:33 smithi026 ceph-mon[21867]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:34.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:33 smithi026 ceph-mon[26568]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:36.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:35 smithi151 ceph-mon[22663]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:36.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:35 smithi026 ceph-mon[21867]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:36.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:35 smithi026 ceph-mon[26568]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:37.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:06:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:06:37.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:06:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:36] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-04-27T02:06:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:37 smithi151 ceph-mon[22663]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:38.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:37 smithi026 ceph-mon[21867]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:37 smithi026 ceph-mon[26568]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:40.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:39 smithi151 ceph-mon[22663]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:40.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:39 smithi026 ceph-mon[21867]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:40.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:39 smithi026 ceph-mon[26568]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:42.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:41 smithi151 ceph-mon[22663]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:42.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:41 smithi026 ceph-mon[21867]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:42.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:41 smithi026 ceph-mon[26568]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:42 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:43.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:42 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:42 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:44.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:43 smithi026 ceph-mon[21867]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:44.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:43 smithi026 ceph-mon[26568]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:44.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:43 smithi151 ceph-mon[22663]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:46.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:45 smithi151 ceph-mon[22663]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:46.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:45 smithi026 ceph-mon[21867]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:46.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:45 smithi026 ceph-mon[26568]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:47.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:06:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:06:47.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:06:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:46] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-04-27T02:06:48.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:47 smithi151 ceph-mon[22663]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:48.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:47 smithi026 ceph-mon[21867]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:47 smithi026 ceph-mon[26568]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:50.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:49 smithi151 ceph-mon[22663]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:50.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:49 smithi026 ceph-mon[21867]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:50.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:49 smithi026 ceph-mon[26568]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:52.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:51 smithi026 ceph-mon[21867]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:52.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:51 smithi026 ceph-mon[26568]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:52.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:51 smithi151 ceph-mon[22663]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:53.250 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:52 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:53.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:52 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:53.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:52 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:06:54.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:53 smithi151 ceph-mon[22663]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:54.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:53 smithi026 ceph-mon[21867]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:54.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:53 smithi026 ceph-mon[26568]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:56.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:55 smithi151 ceph-mon[22663]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:55 smithi026 ceph-mon[21867]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:55 smithi026 ceph-mon[26568]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:06:57.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:06:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:06:57.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:06:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:06:56] "GET /metrics HTTP/1.1" 200 208038 "" "Prometheus/2.33.4" 2024-04-27T02:06:58.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:57 smithi151 ceph-mon[22663]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:58.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:57 smithi026 ceph-mon[21867]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:06:58.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:57 smithi026 ceph-mon[26568]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:00.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:06:59 smithi151 ceph-mon[22663]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:00.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:06:59 smithi026 ceph-mon[21867]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:00.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:06:59 smithi026 ceph-mon[26568]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:02.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:01 smithi151 ceph-mon[22663]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:02.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:01 smithi026 ceph-mon[21867]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:02.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:01 smithi026 ceph-mon[26568]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:02 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:07:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:07:03.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:02 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:03.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:07:03.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:07:03.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:02 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:03.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:07:03.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:07:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:03 smithi151 ceph-mon[22663]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:04.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:03 smithi026 ceph-mon[21867]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:04.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:03 smithi026 ceph-mon[26568]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:06.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:05 smithi151 ceph-mon[22663]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:06.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:05 smithi026 ceph-mon[21867]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:06.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:05 smithi026 ceph-mon[26568]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:07:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:07:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:07:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:06 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:07:07.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:07:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:07:07.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:07:07.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:07:07.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:07:07.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:07:07.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:07:07.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:07:07.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:07:07.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:07:07.358 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:07:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:06] "GET /metrics HTTP/1.1" 200 208030 "" "Prometheus/2.33.4" 2024-04-27T02:07:08.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:07 smithi151 ceph-mon[22663]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:08.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:07 smithi026 ceph-mon[21867]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:08.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:07 smithi026 ceph-mon[26568]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:07:10.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:09 smithi151 ceph-mon[22663]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:10.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:09 smithi026 ceph-mon[21867]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:10.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:09 smithi026 ceph-mon[26568]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:12.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:11 smithi151 ceph-mon[22663]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:12.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:11 smithi026 ceph-mon[21867]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:12.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:11 smithi026 ceph-mon[26568]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:13.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:12 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:12 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:13.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:12 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:14.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:13 smithi151 ceph-mon[22663]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:14.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:13 smithi026 ceph-mon[21867]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:14.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:13 smithi026 ceph-mon[26568]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:16.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:15 smithi151 ceph-mon[22663]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:16.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:15 smithi026 ceph-mon[21867]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:16.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:15 smithi026 ceph-mon[26568]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:17.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:07:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:07:17.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:07:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:16] "GET /metrics HTTP/1.1" 200 208030 "" "Prometheus/2.33.4" 2024-04-27T02:07:18.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:17 smithi151 ceph-mon[22663]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:18.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:17 smithi026 ceph-mon[21867]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:18.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:17 smithi026 ceph-mon[26568]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:20.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:19 smithi151 ceph-mon[22663]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:20.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:19 smithi026 ceph-mon[21867]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:20.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:19 smithi026 ceph-mon[26568]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:22.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:21 smithi026 ceph-mon[21867]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:22.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:21 smithi026 ceph-mon[26568]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:22.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:21 smithi151 ceph-mon[22663]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:23.250 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:22 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:23.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:22 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:23.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:22 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:24.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:23 smithi151 ceph-mon[22663]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:24.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:23 smithi026 ceph-mon[26568]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:24.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:23 smithi026 ceph-mon[21867]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:26.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:25 smithi151 ceph-mon[22663]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:25 smithi026 ceph-mon[21867]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:26.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:25 smithi026 ceph-mon[26568]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:27.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:07:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:07:27.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:07:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:26] "GET /metrics HTTP/1.1" 200 208016 "" "Prometheus/2.33.4" 2024-04-27T02:07:28.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:27 smithi151 ceph-mon[22663]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:28.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:27 smithi026 ceph-mon[21867]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:28.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:27 smithi026 ceph-mon[26568]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:30.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:29 smithi151 ceph-mon[22663]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:30.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:29 smithi026 ceph-mon[21867]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:30.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:29 smithi026 ceph-mon[26568]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:32.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:31 smithi151 ceph-mon[22663]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:32.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:31 smithi026 ceph-mon[21867]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:32.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:31 smithi026 ceph-mon[26568]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:33.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:32 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:33.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:32 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:33.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:32 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:34.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:33 smithi151 ceph-mon[22663]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:34.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:33 smithi026 ceph-mon[21867]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:34.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:33 smithi026 ceph-mon[26568]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:36.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:35 smithi151 ceph-mon[22663]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:36.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:35 smithi026 ceph-mon[21867]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:36.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:35 smithi026 ceph-mon[26568]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:37.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:07:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:07:37.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:07:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:36] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-04-27T02:07:38.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:37 smithi151 ceph-mon[22663]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:38.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:37 smithi026 ceph-mon[21867]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:38.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:37 smithi026 ceph-mon[26568]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:40.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:39 smithi151 ceph-mon[22663]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:40.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:39 smithi026 ceph-mon[21867]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:40.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:39 smithi026 ceph-mon[26568]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:42.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:41 smithi151 ceph-mon[22663]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:42.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:41 smithi026 ceph-mon[21867]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:42.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:41 smithi026 ceph-mon[26568]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:42 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:43.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:42 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:43.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:42 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:43 smithi151 ceph-mon[22663]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:44.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:43 smithi026 ceph-mon[21867]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:44.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:43 smithi026 ceph-mon[26568]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:46.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:45 smithi151 ceph-mon[22663]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:46.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:45 smithi026 ceph-mon[21867]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:46.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:45 smithi026 ceph-mon[26568]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:47.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:07:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:07:47.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:07:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:46] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-04-27T02:07:48.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:47 smithi151 ceph-mon[22663]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:47 smithi026 ceph-mon[21867]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:47 smithi026 ceph-mon[26568]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:50.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:49 smithi151 ceph-mon[22663]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:50.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:49 smithi026 ceph-mon[21867]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:50.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:49 smithi026 ceph-mon[26568]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:52.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:51 smithi151 ceph-mon[22663]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:52.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:51 smithi026 ceph-mon[21867]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:52.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:51 smithi026 ceph-mon[26568]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:53.250 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:53 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:53.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:53 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:53.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:53 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:07:54.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:54 smithi151 ceph-mon[22663]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:54.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:54 smithi026 ceph-mon[21867]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:54.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:54 smithi026 ceph-mon[26568]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:56.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:56 smithi151 ceph-mon[22663]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:56 smithi026 ceph-mon[21867]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:56.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:56 smithi026 ceph-mon[26568]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:07:57.219 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:07:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:56] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-04-27T02:07:57.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:07:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:07:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:07:57.616 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-27T02:07:58.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:07:58 smithi026 ceph-mon[21867]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:07:58 smithi026 ceph-mon[26568]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:07:58.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:07:58 smithi151 ceph-mon[22663]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:00.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:00 smithi026 ceph-mon[21867]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:00.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:00 smithi026 ceph-mon[26568]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:00.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-27T02:08:00.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:00 smithi151 ceph-mon[22663]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:02.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:02 smithi026 ceph-mon[21867]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:02.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:02 smithi026 ceph-mon[26568]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:02.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:02 smithi151 ceph-mon[22663]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:03 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:08:03.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:03 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:08:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:03 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:08:03.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:03 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:08:03.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-27T02:08:03.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:03 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:08:03.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:03 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:08:04.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:04 smithi026 ceph-mon[21867]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:04 smithi151 ceph-mon[22663]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:04.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:04 smithi026 ceph-mon[26568]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:05.914 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:08:06.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:06 smithi026 ceph-mon[21867]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:06.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:06 smithi026 ceph-mon[26568]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:06.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:06 smithi151 ceph-mon[22663]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:08:07.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:07 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:08:07.226 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:08:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:06] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-04-27T02:08:07.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:08:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:08:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:07 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:08:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:07 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:08:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:07 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:08:07.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:07 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 3m ago 3m 13.3M - ba2b418f427c 5cec00038ac8 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (3m) 3m ago 3m 46.6M - 8.3.5 dad864ee21e9 611ca410ba26 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (3m) 3m ago 3m 67.9M - 3.5 e1d6a67b021e 75d6a715b37b 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e b0459dc8195a 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 56112248aa10 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (10m) 3m ago 10m 44.6M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:08:07.992 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (8m) 3m ago 8m 35.6M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (8m) 3m ago 8m 35.5M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 3m ago 4m 9021k - 1dbe0e931976 5405a3c0085a 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (4m) 3m ago 4m 15.5M - 1dbe0e931976 8e60a39d89ed 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) 3m ago 8m 48.1M 4017M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (7m) 3m ago 7m 49.1M 4017M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) 3m ago 7m 44.1M 4017M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (7m) 3m ago 7m 48.4M 4017M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (6m) 3m ago 6m 47.2M 4272M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (6m) 3m ago 6m 47.3M 4272M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (5m) 3m ago 5m 43.7M 4272M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (5m) 3m ago 5m 46.1M 4272M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (3m) 3m ago 4m 43.8M - 514e6a882f6e 6c19ce81f41e 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:08:07.993 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (3m) 3m ago 3m 71.6M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:08:08.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:08 smithi151 ceph-mon[22663]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:08.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:08 smithi151 ceph-mon[22663]: from='client.24862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:08.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:08 smithi026 ceph-mon[21867]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:08.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:08 smithi026 ceph-mon[21867]: from='client.24862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:08.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:08 smithi026 ceph-mon[26568]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:08.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:08 smithi026 ceph-mon[26568]: from='client.24862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:08.638 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-27T02:08:09.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:09 smithi026 ceph-mon[21867]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:09.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:09 smithi026 ceph-mon[26568]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:09.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:09 smithi151 ceph-mon[22663]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:10.646 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:08:10.647 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-27T02:08:10.648 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:08:10.648 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:08:11.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:11 smithi026 ceph-mon[21867]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:11.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:11 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3726175911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:08:11.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:11 smithi026 ceph-mon[26568]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:11.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:11 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3726175911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:08:11.323 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph -s' 2024-04-27T02:08:11.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:11 smithi151 ceph-mon[22663]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:11.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:11 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3726175911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:08:12.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:12 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:12.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:12 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:12.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:12 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:13.398 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-27T02:08:13.398 INFO:teuthology.orchestra.run.smithi026.stdout: id: 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:08:13.398 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_OK 2024-04-27T02:08:13.398 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:08:13.398 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: y(active, since 5m), standbys: x 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 241 objects, 585 KiB 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 80 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-27T02:08:13.399 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:08:13.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:13 smithi026 ceph-mon[21867]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:13.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:13 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1851172771' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:08:13.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:13 smithi026 ceph-mon[26568]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:13.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:13 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1851172771' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:08:14.024 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ls' 2024-04-27T02:08:14.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:13 smithi151 ceph-mon[22663]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:14.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:13 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1851172771' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:08:15.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:15 smithi026 ceph-mon[21867]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:15.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:15 smithi026 ceph-mon[26568]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi026=a;count:1 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:grafana ?:3000 1/1 3m ago 5m smithi151=a;count:1 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:mgr 2/2 3m ago 8m smithi026=y;smithi151=x;count:2 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:mon 3/3 3m ago 9m smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi151:172.21.15.151=b;count:3 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi026=a;smithi151=b;count:2 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:osd 8 3m ago - 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus ?:9095 1/1 3m ago 5m smithi151=a;count:1 2024-04-27T02:08:16.025 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-27T02:08:16.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:15 smithi151 ceph-mon[22663]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:16.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-27T02:08:17.230 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:08:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:16] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-04-27T02:08:17.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:08:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:08:18.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:17 smithi151 ceph-mon[22663]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:18.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:17 smithi151 ceph-mon[22663]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:18.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:17 smithi026 ceph-mon[21867]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:18.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:17 smithi026 ceph-mon[21867]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:18.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:17 smithi026 ceph-mon[26568]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:18.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:17 smithi026 ceph-mon[26568]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:19.065 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled to redeploy mgr.x on host 'smithi151' 2024-04-27T02:08:19.079 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:18 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3540895161' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:08:19.079 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:18 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3540895161' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:08:19.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:18 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3540895161' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:08:19.735 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps --refresh' 2024-04-27T02:08:20.051 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:19 smithi026 ceph-mon[21867]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:20.051 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:19 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:08:20.051 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:19 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:08:20.051 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:19 smithi026 ceph-mon[26568]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:20.051 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:19 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:08:20.051 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:19 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:08:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:19 smithi151 ceph-mon[22663]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:19 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:08:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:19 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:08:21.076 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:20 smithi026 ceph-mon[21867]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:21.076 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:20 smithi026 ceph-mon[21867]: Schedule redeploy daemon mgr.x 2024-04-27T02:08:21.076 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:20 smithi026 ceph-mon[26568]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:21.076 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:20 smithi026 ceph-mon[26568]: Schedule redeploy daemon mgr.x 2024-04-27T02:08:21.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:20 smithi151 ceph-mon[22663]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:21.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:20 smithi151 ceph-mon[22663]: Schedule redeploy daemon mgr.x 2024-04-27T02:08:21.820 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:08:21.820 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 3m ago 4m 13.3M - ba2b418f427c 5cec00038ac8 2024-04-27T02:08:21.821 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (3m) 3m ago 3m 46.6M - 8.3.5 dad864ee21e9 611ca410ba26 2024-04-27T02:08:21.821 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (3m) 3m ago 3m 67.9M - 3.5 e1d6a67b021e 75d6a715b37b 2024-04-27T02:08:21.821 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e b0459dc8195a 2024-04-27T02:08:21.821 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 56112248aa10 2024-04-27T02:08:21.821 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (10m) 3m ago 10m 44.6M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:08:21.821 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (9m) 3m ago 9m 35.6M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:08:21.821 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (9m) 3m ago 9m 35.5M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 3m ago 4m 9021k - 1dbe0e931976 5405a3c0085a 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (4m) 3m ago 4m 15.5M - 1dbe0e931976 8e60a39d89ed 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) 3m ago 8m 48.1M 4017M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (8m) 3m ago 8m 49.1M 4017M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) 3m ago 7m 44.1M 4017M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (7m) 3m ago 7m 48.4M 4017M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (6m) 3m ago 6m 47.2M 4272M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:08:21.822 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (6m) 3m ago 6m 47.3M 4272M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:08:21.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (6m) 3m ago 6m 43.7M 4272M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:08:21.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (5m) 3m ago 5m 46.1M 4272M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:08:21.823 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (3m) 3m ago 5m 43.8M - 514e6a882f6e 6c19ce81f41e 2024-04-27T02:08:21.823 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:08:21.823 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (3m) 3m ago 3m 71.6M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:08:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:21 smithi151 ceph-mon[22663]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:22.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:21 smithi026 ceph-mon[21867]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:22.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:21 smithi026 ceph-mon[26568]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:22.560 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 180' 2024-04-27T02:08:23.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:22 smithi151 ceph-mon[22663]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:23.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:22 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:23.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:22 smithi026 ceph-mon[21867]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:23.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:22 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:23.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:22 smithi026 ceph-mon[26568]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:08:23.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:22 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:24.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:23 smithi151 ceph-mon[22663]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:24.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:23 smithi026 ceph-mon[21867]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:24.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:23 smithi026 ceph-mon[26568]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:26.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:25 smithi151 ceph-mon[22663]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:26.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:25 smithi026 ceph-mon[21867]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:26.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:25 smithi026 ceph-mon[26568]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:27.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:08:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:08:27.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:08:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:26] "GET /metrics HTTP/1.1" 200 207974 "" "Prometheus/2.33.4" 2024-04-27T02:08:28.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:27 smithi151 ceph-mon[22663]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:28.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:27 smithi026 ceph-mon[21867]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:28.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:27 smithi026 ceph-mon[26568]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:30.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:29 smithi151 ceph-mon[22663]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:30.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:29 smithi026 ceph-mon[26568]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:30.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:29 smithi026 ceph-mon[21867]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:32.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:31 smithi151 ceph-mon[22663]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:32.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:31 smithi026 ceph-mon[21867]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:32.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:31 smithi026 ceph-mon[26568]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:33.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:32 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:33.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:32 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:33.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:32 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:34.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:33 smithi026 ceph-mon[21867]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:34.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:33 smithi026 ceph-mon[26568]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:34.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:33 smithi151 ceph-mon[22663]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:36.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:35 smithi026 ceph-mon[21867]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:36.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:35 smithi026 ceph-mon[26568]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:36.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:35 smithi151 ceph-mon[22663]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:37.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:08:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:08:37.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:08:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:36] "GET /metrics HTTP/1.1" 200 207969 "" "Prometheus/2.33.4" 2024-04-27T02:08:38.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:37 smithi026 ceph-mon[21867]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:37 smithi026 ceph-mon[26568]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:38.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:37 smithi151 ceph-mon[22663]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:40.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:39 smithi151 ceph-mon[22663]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:40.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:39 smithi026 ceph-mon[21867]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:40.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:39 smithi026 ceph-mon[26568]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:42.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:41 smithi151 ceph-mon[22663]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:42.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:41 smithi026 ceph-mon[26568]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:42.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:41 smithi026 ceph-mon[21867]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:42 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:43.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:42 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:43.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:42 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:43 smithi151 ceph-mon[22663]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:44.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:43 smithi026 ceph-mon[21867]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:44.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:43 smithi026 ceph-mon[26568]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:46.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:45 smithi151 ceph-mon[22663]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:46.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:45 smithi026 ceph-mon[21867]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:46.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:45 smithi026 ceph-mon[26568]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:47.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:08:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:08:47.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:08:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:46] "GET /metrics HTTP/1.1" 200 207969 "" "Prometheus/2.33.4" 2024-04-27T02:08:48.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:47 smithi151 ceph-mon[22663]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:47 smithi026 ceph-mon[21867]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:48.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:47 smithi026 ceph-mon[26568]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:50.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:49 smithi151 ceph-mon[22663]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:50.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:49 smithi026 ceph-mon[26568]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:50.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:49 smithi026 ceph-mon[21867]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:52.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:51 smithi026 ceph-mon[21867]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:52.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:51 smithi026 ceph-mon[26568]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:52.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:51 smithi151 ceph-mon[22663]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:53.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:52 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:53.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:52 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:53.250 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:52 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:08:54.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:53 smithi151 ceph-mon[22663]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:54.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:53 smithi026 ceph-mon[21867]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:54.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:53 smithi026 ceph-mon[26568]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:56.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:55 smithi151 ceph-mon[22663]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:55 smithi026 ceph-mon[21867]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:56.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:55 smithi026 ceph-mon[26568]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:08:57.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:08:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:08:57.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:08:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:08:56] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-04-27T02:08:58.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:57 smithi151 ceph-mon[22663]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:58.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:57 smithi026 ceph-mon[21867]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:08:58.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:57 smithi026 ceph-mon[26568]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:00.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:08:59 smithi151 ceph-mon[22663]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:00.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:08:59 smithi026 ceph-mon[21867]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:00.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:08:59 smithi026 ceph-mon[26568]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:02.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:01 smithi151 ceph-mon[22663]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:02.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:01 smithi026 ceph-mon[21867]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:02.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:01 smithi026 ceph-mon[26568]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:03.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:09:03.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:09:03.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:09:03.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:09:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:09:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:09:04.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:03 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:04.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:03 smithi026 ceph-mon[21867]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:04.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:03 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:04.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:03 smithi026 ceph-mon[26568]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:03 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:03 smithi151 ceph-mon[22663]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:06.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:05 smithi151 ceph-mon[22663]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:06.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:05 smithi026 ceph-mon[21867]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:06.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:05 smithi026 ceph-mon[26568]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:07.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:09:07.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:09:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:06] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-04-27T02:09:08.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:07 smithi151 ceph-mon[22663]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:08.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:07 smithi026 ceph-mon[21867]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:08.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:07 smithi026 ceph-mon[26568]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:09 smithi151 ceph-mon[22663]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:10.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:09 smithi026 ceph-mon[21867]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:10.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:09 smithi026 ceph-mon[26568]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:12.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:11 smithi151 ceph-mon[22663]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:12.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:11 smithi026 ceph-mon[21867]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:12.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:11 smithi026 ceph-mon[26568]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:14.164 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:13 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:14.164 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:13 smithi026 ceph-mon[21867]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:14.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:13 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:14.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:13 smithi026 ceph-mon[26568]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:14.229 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:13 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:14.229 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:13 smithi151 ceph-mon[22663]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:15.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:15 smithi151 ceph-mon[22663]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:15.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:15 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:15 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:15.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:15 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:09:15.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:15 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:15 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[21867]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[26568]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:15 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:17.098 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:09:17.107 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:09:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:16] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-04-27T02:09:17.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:17 smithi151 ceph-mon[22663]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:17.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:17 smithi026 ceph-mon[21867]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:17.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:17 smithi026 ceph-mon[26568]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:19.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:18 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:18 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:18 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:18 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:09:19.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:18 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:09:19.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:18 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:19.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:09:19.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:09:19.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:19.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:19.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:09:19.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:09:19.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:18 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:19 smithi151 ceph-mon[22663]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:19 smithi151 ceph-mon[22663]: Deploying daemon mgr.x on smithi151 2024-04-27T02:09:20.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:19 smithi026 ceph-mon[21867]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:20.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:19 smithi026 ceph-mon[21867]: Deploying daemon mgr.x on smithi151 2024-04-27T02:09:20.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:19 smithi026 ceph-mon[26568]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:20.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:19 smithi026 ceph-mon[26568]: Deploying daemon mgr.x on smithi151 2024-04-27T02:09:22.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:21 smithi026 ceph-mon[21867]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:22.042 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:21 smithi026 ceph-mon[26568]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:21 smithi151 ceph-mon[22663]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:23.829 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:09:23.829 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:09:24.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:23 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:24.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:23 smithi151 ceph-mon[22663]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:24.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:23 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:24.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:23 smithi026 ceph-mon[21867]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:24.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:23 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:24.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:23 smithi026 ceph-mon[26568]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:26.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:25 smithi151 ceph-mon[22663]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:26.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:25 smithi026 ceph-mon[21867]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:26.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:25 smithi026 ceph-mon[26568]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:27.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:09:27.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:09:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:26] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-04-27T02:09:28.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:27 smithi151 ceph-mon[22663]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:28.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:27 smithi026 ceph-mon[21867]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:28.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:27 smithi026 ceph-mon[26568]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:30.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:29 smithi151 ceph-mon[22663]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:30.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:29 smithi026 ceph-mon[21867]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:30.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:29 smithi026 ceph-mon[26568]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:32.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:31 smithi151 ceph-mon[22663]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:32.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:31 smithi026 ceph-mon[21867]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:32.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:31 smithi026 ceph-mon[26568]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:33.840 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:09:33.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:09:33.840 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:09:33.840 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:09:34.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:33 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:34.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:33 smithi151 ceph-mon[22663]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:34.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:33 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:34.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:33 smithi026 ceph-mon[26568]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:34.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:33 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:34.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:33 smithi026 ceph-mon[21867]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:36.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:35 smithi151 ceph-mon[22663]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:36.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:35 smithi026 ceph-mon[21867]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:36.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:35 smithi026 ceph-mon[26568]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:37.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:09:37.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:09:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:36] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-04-27T02:09:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:37 smithi151 ceph-mon[22663]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:38.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:37 smithi026 ceph-mon[21867]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:37 smithi026 ceph-mon[26568]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:40.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:39 smithi151 ceph-mon[22663]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:40.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:39 smithi026 ceph-mon[21867]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:40.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:39 smithi026 ceph-mon[26568]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:42.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:41 smithi026 ceph-mon[21867]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:42.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:41 smithi026 ceph-mon[26568]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:42.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:41 smithi151 ceph-mon[22663]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:43.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:09:43.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:09:43.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:43.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:09:43.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:43.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:09:44.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:43 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:44.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:43 smithi026 ceph-mon[21867]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:44.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:43 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:44.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:43 smithi026 ceph-mon[26568]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:43 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:43 smithi151 ceph-mon[22663]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:46.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:45 smithi026 ceph-mon[21867]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:46.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:45 smithi026 ceph-mon[26568]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:46.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:45 smithi151 ceph-mon[22663]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:47.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[23844]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:09:47.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:09:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:46] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-04-27T02:09:48.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:47 smithi026 ceph-mon[21867]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:48.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:47 smithi026 ceph-mon[26568]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:48.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:47 smithi151 ceph-mon[22663]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:50.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:49 smithi151 ceph-mon[22663]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:50.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:49 smithi026 ceph-mon[21867]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:50.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:49 smithi026 ceph-mon[26568]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:52.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:51 smithi026 ceph-mon[21867]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:52.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:51 smithi026 ceph-mon[26568]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:52.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:51 smithi151 ceph-mon[22663]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:09:53.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:09:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:09:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:09:54.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:53 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:54.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:53 smithi151 ceph-mon[22663]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:54.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:53 smithi151 systemd[1]: Stopping Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:09:54.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 bash[42875]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-27T02:09:54.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:53 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:54.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:53 smithi026 ceph-mon[21867]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:54.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:53 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:09:54.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:53 smithi026 ceph-mon[26568]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:54.424 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 podman[42888]: 2024-04-27 02:09:54.157017478 +0000 UTC m=+0.130686067 container died b0459dc8195a588e31dcb378e7006290584232df0819af79ab6d116bc8e425f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vendor=Red Hat, Inc., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, version=8, ceph=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8) 2024-04-27T02:09:54.783 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 podman[42888]: 2024-04-27 02:09:54.545186458 +0000 UTC m=+0.518855055 container cleanup b0459dc8195a588e31dcb378e7006290584232df0819af79ab6d116bc8e425f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, distribution-scope=public, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, release=754, version=8, ceph=True, RELEASE=HEAD, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-27T02:09:54.783 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 bash[42888]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x 2024-04-27T02:09:54.783 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 bash[42915]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-27T02:09:55.098 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 podman[42902]: 2024-04-27 02:09:54.805314212 +0000 UTC m=+0.644147122 container remove b0459dc8195a588e31dcb378e7006290584232df0819af79ab6d116bc8e425f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vendor=Red Hat, Inc., version=8, GIT_CLEAN=True, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-27T02:09:55.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:09:55.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-27T02:09:55.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 systemd[1]: Stopped Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:09:55.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:54 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service: Consumed 17.615s CPU time. 2024-04-27T02:09:55.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 systemd[1]: Starting Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:09:55.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:09:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:09:54.811Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:09:55.455 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 podman[43019]: 2024-04-27 02:09:55.127373646 +0000 UTC m=+0.018420305 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:09:55.455 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 podman[43019]: 2024-04-27 02:09:55.248318879 +0000 UTC m=+0.139365509 container create 210983b6ef1956da086775b8f817e220629e55583d6818a4c5828ded37c3cbe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:09:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 podman[43019]: 2024-04-27 02:09:55.476943887 +0000 UTC m=+0.367990514 container init 210983b6ef1956da086775b8f817e220629e55583d6818a4c5828ded37c3cbe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-27T02:09:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 podman[43019]: 2024-04-27 02:09:55.48026361 +0000 UTC m=+0.371310241 container start 210983b6ef1956da086775b8f817e220629e55583d6818a4c5828ded37c3cbe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:09:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-mgr[43038]: -- 172.21.15.151:0/1736184796 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cc9ab885a0 con 0x55cc9ab5cc00 2024-04-27T02:09:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 bash[43019]: 210983b6ef1956da086775b8f817e220629e55583d6818a4c5828ded37c3cbe5 2024-04-27T02:09:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 systemd[1]: Started Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:09:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:55.634+0000 7f8137743200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:09:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:55.835+0000 7f8137743200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:09:56.098 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:55.953+0000 7f8137743200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:09:56.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-mon[22663]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:56.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:56.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:56.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:09:56.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:56.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:55 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:09:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[21867]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:09:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:56.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:09:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[26568]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:09:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:09:56.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:55 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:09:56.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:56.274+0000 7f8137743200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:09:56.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:56.364+0000 7f8137743200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:09:57.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:09:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:09:56] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-04-27T02:09:57.895 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:57 smithi151 ceph-mon[22663]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:57.895 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:57.657+0000 7f8137743200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:09:57.895 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:57.838+0000 7f8137743200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:09:58.298 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:58.013+0000 7f8137743200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:09:58.298 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:58.319+0000 7f8137743200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:09:58.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:57 smithi026 ceph-mon[21867]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:58.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:57 smithi026 ceph-mon[26568]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:09:58.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:58.412+0000 7f8137743200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:09:59.098 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:58.619+0000 7f8137743200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:09:59.098 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:58.816+0000 7f8137743200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:09:59.694 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:59.451+0000 7f8137743200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:09:59.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:59 smithi151 ceph-mon[22663]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:09:59.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:59 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:59.973 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:09:59 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:09:59.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:59.897+0000 7f8137743200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:10:00.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:09:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:09:59.993+0000 7f8137743200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:10:00.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:00 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:00.199+0000 7f8137743200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:10:00.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:59 smithi026 ceph-mon[21867]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:00.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:59 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:10:00.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:09:59 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:10:00.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:59 smithi026 ceph-mon[26568]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:00.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:59 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:10:00.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:09:59 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:10:01.058 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:00 smithi151 ceph-mon[22663]: overall HEALTH_OK 2024-04-27T02:10:01.058 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:00 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:00.733+0000 7f8137743200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:10:01.058 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:00 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:00.827+0000 7f8137743200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:10:01.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:01.079+0000 7f8137743200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:10:01.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:01.215+0000 7f8137743200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:10:01.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:01.311+0000 7f8137743200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:10:01.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:00 smithi026 ceph-mon[21867]: overall HEALTH_OK 2024-04-27T02:10:01.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:00 smithi026 ceph-mon[26568]: overall HEALTH_OK 2024-04-27T02:10:01.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:01.426+0000 7f8137743200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:10:01.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:01.519+0000 7f8137743200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:10:02.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:01 smithi151 ceph-mon[22663]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:02.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:01 smithi026 ceph-mon[26568]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:02.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:01 smithi026 ceph-mon[21867]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:02.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:02.332+0000 7f8137743200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:10:02.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:02.549+0000 7f8137743200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:10:02.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:10:02] ENGINE Bus STARTING 2024-04-27T02:10:03.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:10:03.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: Standby manager daemon x restarted 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: Standby manager daemon x started 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CherryPy Checker: 2024-04-27T02:10:03.099 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: The Application mounted at '' has an empty config. 2024-04-27T02:10:03.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:10:03.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:10:02] ENGINE Serving on http://:::9283 2024-04-27T02:10:03.100 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:10:02] ENGINE Bus STARTED 2024-04-27T02:10:03.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:10:03.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:10:03.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: Standby manager daemon x restarted 2024-04-27T02:10:03.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T02:10:03.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:10:03.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:10:03.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: Standby manager daemon x started 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:10:03.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:10:03.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:02 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:10:03.488 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:03.159Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:03.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:10:03.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:03.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:03.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:03 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:03 smithi151 ceph-mon[22663]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:03 smithi151 ceph-mon[22663]: mgrmap e22: y(active, since 7m), standbys: x 2024-04-27T02:10:04.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:04.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-mon[21867]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:04.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-mon[21867]: mgrmap e22: y(active, since 7m), standbys: x 2024-04-27T02:10:04.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:04.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-mon[26568]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:04.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:03 smithi026 ceph-mon[26568]: mgrmap e22: y(active, since 7m), standbys: x 2024-04-27T02:10:06.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:05 smithi151 ceph-mon[22663]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:06.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:05 smithi026 ceph-mon[21867]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:06.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:05 smithi026 ceph-mon[26568]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:07.098 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:10:07.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:10:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:06] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-04-27T02:10:08.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:07 smithi151 ceph-mon[22663]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:08.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:07 smithi026 ceph-mon[21867]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:08.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:07 smithi026 ceph-mon[26568]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:09 smithi151 ceph-mon[22663]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:10.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:09 smithi026 ceph-mon[21867]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:10.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:09 smithi026 ceph-mon[26568]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:12.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:11 smithi151 ceph-mon[22663]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:12.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:11 smithi026 ceph-mon[21867]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:12.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:11 smithi026 ceph-mon[26568]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:13.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:10:13.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:13.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:13.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:13.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:14.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:13 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:14.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:13 smithi151 ceph-mon[22663]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:14.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:13 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:14.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:13 smithi026 ceph-mon[21867]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:14.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:13 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:14.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:13 smithi026 ceph-mon[26568]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:16.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:15 smithi151 ceph-mon[22663]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:16.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:15 smithi026 ceph-mon[21867]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:16.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:15 smithi026 ceph-mon[26568]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:17.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:10:17.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:10:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:16] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-04-27T02:10:18.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:17 smithi151 ceph-mon[22663]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:18.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:17 smithi026 ceph-mon[21867]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:18.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:17 smithi026 ceph-mon[26568]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:20.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:19 smithi151 ceph-mon[22663]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:20.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:19 smithi026 ceph-mon[21867]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:20.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:19 smithi026 ceph-mon[26568]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:22.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:21 smithi026 ceph-mon[21867]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:22.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:21 smithi026 ceph-mon[26568]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:22.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:21 smithi151 ceph-mon[22663]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:23.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:10:23.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:23.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:23.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:24.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:23 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:24.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:23 smithi151 ceph-mon[22663]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:24.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:23 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:24.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:23 smithi026 ceph-mon[21867]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:24.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:23 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:24.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:23 smithi026 ceph-mon[26568]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:26.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:25 smithi151 ceph-mon[22663]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:25 smithi026 ceph-mon[21867]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:26.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:25 smithi026 ceph-mon[26568]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:27.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:10:27.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:10:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:26] "GET /metrics HTTP/1.1" 200 207997 "" "Prometheus/2.33.4" 2024-04-27T02:10:28.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:27 smithi151 ceph-mon[22663]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:28.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:27 smithi026 ceph-mon[21867]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:28.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:27 smithi026 ceph-mon[26568]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:30.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:29 smithi151 ceph-mon[22663]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:30.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:29 smithi026 ceph-mon[21867]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:30.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:29 smithi026 ceph-mon[26568]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:32.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:31 smithi151 ceph-mon[22663]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:32.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:31 smithi026 ceph-mon[21867]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:32.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:31 smithi026 ceph-mon[26568]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:33.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:10:33.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:33.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:33.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:34.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:33 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:34.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:33 smithi151 ceph-mon[22663]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:34.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:33 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:34.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:33 smithi026 ceph-mon[21867]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:34.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:33 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:34.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:33 smithi026 ceph-mon[26568]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:36.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:35 smithi151 ceph-mon[22663]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:36.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:35 smithi026 ceph-mon[21867]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:36.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:35 smithi026 ceph-mon[26568]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:37.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:10:37.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:10:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:36] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-04-27T02:10:38.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:37 smithi151 ceph-mon[22663]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:38.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:37 smithi026 ceph-mon[21867]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:38.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:37 smithi026 ceph-mon[26568]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:40.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:39 smithi151 ceph-mon[22663]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:40.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:39 smithi026 ceph-mon[21867]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:40.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:39 smithi026 ceph-mon[26568]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:42.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:41 smithi151 ceph-mon[22663]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:42.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:41 smithi026 ceph-mon[21867]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:42.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:41 smithi026 ceph-mon[26568]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:43.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:10:43.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:43.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:43.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:44.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:43 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:43 smithi151 ceph-mon[22663]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:44.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:43 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:44.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:43 smithi026 ceph-mon[21867]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:44.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:43 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:44.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:43 smithi026 ceph-mon[26568]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:46.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:45 smithi151 ceph-mon[22663]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:46.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:45 smithi026 ceph-mon[21867]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:46.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:45 smithi026 ceph-mon[26568]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:47.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:10:47.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:10:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:46] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-04-27T02:10:48.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:47 smithi151 ceph-mon[22663]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:47 smithi026 ceph-mon[21867]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:48.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:47 smithi026 ceph-mon[26568]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:50.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:49 smithi151 ceph-mon[22663]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:50.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:49 smithi026 ceph-mon[21867]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:50.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:49 smithi026 ceph-mon[26568]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:52.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:51 smithi151 ceph-mon[22663]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:52.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:51 smithi026 ceph-mon[21867]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:52.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:51 smithi026 ceph-mon[26568]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:10:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:10:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:10:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:10:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:10:54.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:53 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:54.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:53 smithi151 ceph-mon[22663]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:54.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:53 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:54.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:53 smithi026 ceph-mon[21867]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:54.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:53 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:10:54.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:53 smithi026 ceph-mon[26568]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:56.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:55 smithi151 ceph-mon[22663]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:56.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:55 smithi026 ceph-mon[21867]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:56.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:55 smithi026 ceph-mon[26568]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:10:57.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:10:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:10:57.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:10:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:10:56] "GET /metrics HTTP/1.1" 200 207998 "" "Prometheus/2.33.4" 2024-04-27T02:10:58.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:57 smithi151 ceph-mon[22663]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:58.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:57 smithi026 ceph-mon[21867]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:10:58.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:57 smithi026 ceph-mon[26568]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:00.035 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:59 smithi151 ceph-mon[22663]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:00.036 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:59 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:11:00.036 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:59 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:11:00.036 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:10:59 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:11:00.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[26568]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:00.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:11:00.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:11:00.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:11:00.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[21867]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:00.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:11:00.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:11:00.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:10:59 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:11:01.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:01 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:11:01.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:01 smithi151 ceph-mon[22663]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:01.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:01 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:11:01.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:01 smithi026 ceph-mon[21867]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:01.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:01 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' 2024-04-27T02:11:01.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:01 smithi026 ceph-mon[26568]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:02.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:02.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:02 smithi151 ceph-mon[22663]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:02.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:02.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:02 smithi026 ceph-mon[21867]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:02.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:02.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:02 smithi026 ceph-mon[26568]: from='mgr.24337 172.21.15.26:0/2736664002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:03.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:11:03.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:03.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:03.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:03.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:03.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:11:04.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:03 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:04.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:03 smithi026 ceph-mon[21867]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:04.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:03 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:04.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:03 smithi026 ceph-mon[26568]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:03 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:04.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:03 smithi151 ceph-mon[22663]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:06.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:05 smithi026 ceph-mon[21867]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:06.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:05 smithi026 ceph-mon[26568]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:06.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:05 smithi151 ceph-mon[22663]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:07.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:11:07.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:06] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-04-27T02:11:08.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:07 smithi151 ceph-mon[22663]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:08.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:07 smithi026 ceph-mon[21867]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:08.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:07 smithi026 ceph-mon[26568]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:09 smithi151 ceph-mon[22663]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:10.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:09 smithi026 ceph-mon[21867]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:10.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:09 smithi026 ceph-mon[26568]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:11 smithi026 ceph-mon[21867]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:12.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:11 smithi026 ceph-mon[26568]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:12.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:11 smithi151 ceph-mon[22663]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:13.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:11:13.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:13.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:13.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:13.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:13.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:11:14.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:13 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:14.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:13 smithi151 ceph-mon[22663]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:14.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:13 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:14.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:13 smithi026 ceph-mon[21867]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:14.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:13 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:14.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:13 smithi026 ceph-mon[26568]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:16.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:15 smithi151 ceph-mon[22663]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:15 smithi026 ceph-mon[21867]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:16.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:15 smithi026 ceph-mon[26568]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:17.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:11:17.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:16] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-04-27T02:11:18.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:17 smithi151 ceph-mon[22663]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:18.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:17 smithi026 ceph-mon[21867]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:18.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:17 smithi026 ceph-mon[26568]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:20.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:19 smithi151 ceph-mon[22663]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:20.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:19 smithi026 ceph-mon[21867]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:20.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:19 smithi026 ceph-mon[26568]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:22.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:21 smithi026 ceph-mon[21867]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:22.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:21 smithi026 ceph-mon[26568]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:22.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:21 smithi151 ceph-mon[22663]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:23.856 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:11:23.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:11:23.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:23.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:11:24.150 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:23 smithi151 ceph-mon[22663]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:24.150 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:23 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:24.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:23 smithi026 ceph-mon[21867]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:24.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:23 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:24.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:23 smithi026 ceph-mon[26568]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:24.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:23 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:25.276 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:11:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:25 smithi151 ceph-mon[22663]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:25 smithi026 ceph-mon[21867]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:26.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:25 smithi026 ceph-mon[26568]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:27.062 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:26] "GET /metrics HTTP/1.1" 200 208001 "" "Prometheus/2.33.4" 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) 2m ago 7m 25.0M - ba2b418f427c 5cec00038ac8 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (7m) 87s ago 7m 54.3M - 8.3.5 dad864ee21e9 611ca410ba26 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (6m) 2m ago 6m 53.1M - 3.5 e1d6a67b021e 75d6a715b37b 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283 running (92s) 87s ago 11m 42.4M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 210983b6ef19 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (13m) 2m ago 13m 461M - 17.2.0 e1d6a67b021e 56112248aa10 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (13m) 2m ago 13m 48.1M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:11:27.289 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (12m) 87s ago 12m 38.1M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (12m) 2m ago 12m 36.9M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (8m) 2m ago 8m 15.5M - 1dbe0e931976 5405a3c0085a 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (7m) 87s ago 7m 17.5M - 1dbe0e931976 8e60a39d89ed 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (11m) 2m ago 11m 50.1M 4017M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (11m) 2m ago 11m 51.1M 4017M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (10m) 2m ago 10m 47.2M 4017M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (10m) 2m ago 10m 52.2M 4017M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (10m) 87s ago 10m 50.9M 4272M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (9m) 87s ago 9m 51.3M 4272M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (9m) 87s ago 9m 47.1M 4272M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (8m) 87s ago 8m 49.9M 4272M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (6m) 87s ago 8m 68.5M - 514e6a882f6e 6c19ce81f41e 2024-04-27T02:11:27.290 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (6m) 2m ago 6m 71.4M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:11:27.291 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (6m) 87s ago 6m 73.2M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:11:27.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:11:27.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-27T02:11:27.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:27 smithi026 ceph-mon[21867]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:27 smithi026 ceph-mon[26568]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:28.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:27 smithi151 ceph-mon[22663]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:29.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:28 smithi026 ceph-mon[21867]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:11:29.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:28 smithi026 ceph-mon[26568]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:11:29.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:28 smithi151 ceph-mon[22663]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:11:30.010 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:11:30.011 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:11:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:29 smithi151 ceph-mon[22663]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:29 smithi026 ceph-mon[21867]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:29 smithi026 ceph-mon[26568]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:30.687 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph -s' 2024-04-27T02:11:31.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:30 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/107483797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:11:31.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:30 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/107483797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:11:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:30 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/107483797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:11:32.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:31 smithi026 ceph-mon[21867]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:32.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:31 smithi026 ceph-mon[26568]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:32.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:31 smithi151 ceph-mon[22663]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:32.814 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-27T02:11:32.814 INFO:teuthology.orchestra.run.smithi026.stdout: id: 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:11:32.814 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_OK 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: y(active, since 8m), standbys: x 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 241 objects, 585 KiB 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 80 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:11:32.815 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-27T02:11:32.816 INFO:teuthology.orchestra.run.smithi026.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-27T02:11:32.816 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:11:33.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:32 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/707999795' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:11:33.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:32 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/707999795' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:11:33.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:32 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/707999795' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:11:33.481 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:11:33.794 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:11:33.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:33.794 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:33.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:33.794 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:33.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:11:34.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:33 smithi026 ceph-mon[26568]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:34.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:33 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:34.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:33 smithi026 ceph-mon[21867]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:34.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:33 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:34.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:33 smithi151 ceph-mon[22663]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:34.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:33 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:35.556 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:11:36.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:35 smithi026 ceph-mon[26568]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:36.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:35 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3168004430' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:11:36.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:35 smithi026 ceph-mon[21867]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:36.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:35 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3168004430' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:11:36.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:35 smithi151 ceph-mon[22663]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:36.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:35 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3168004430' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:11:36.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-27T02:11:37.014 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:36 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:36] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-04-27T02:11:37.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:11:38.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:37 smithi026 ceph-mon[21867]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:38.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:37 smithi026 ceph-mon[26568]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:38.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:37 smithi151 ceph-mon[22663]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:38.642 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:11:39.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:38 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1110301187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:11:39.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:38 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1110301187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:11:39.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:38 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1110301187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:11:39.480 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph mgr fail' 2024-04-27T02:11:40.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:39 smithi026 ceph-mon[21867]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:40.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:39 smithi026 ceph-mon[26568]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:40.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:39 smithi151 ceph-mon[22663]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:11:42.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[21867]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:42.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4163826670' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:11:42.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:11:42.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[21867]: osdmap e94: 8 total, 8 up, 8 in 2024-04-27T02:11:42.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[26568]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:42.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4163826670' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:11:42.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:11:42.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mon[26568]: osdmap e94: 8 total, 8 up, 8 in 2024-04-27T02:11:42.178 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setuser ceph since I am not root 2024-04-27T02:11:42.178 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ignoring --setgroup ceph since I am not root 2024-04-27T02:11:42.178 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:41 smithi026 ceph-mgr[22100]: -- 172.21.15.26:0/3947258298 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562c99cc4340 con 0x562c99c9d000 2024-04-27T02:11:42.179 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:42.121+0000 7f39d5c77000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:11:42.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:41 smithi151 ceph-mon[22663]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:11:42.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:41 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4163826670' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:11:42.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:41 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:11:42.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:41 smithi151 ceph-mon[22663]: osdmap e94: 8 total, 8 up, 8 in 2024-04-27T02:11:42.284 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:11:41] ENGINE Bus STOPPING 2024-04-27T02:11:42.285 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:11:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:11:42.285 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:11:41] ENGINE Bus STOPPED 2024-04-27T02:11:42.285 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:11:42] ENGINE Bus STARTING 2024-04-27T02:11:42.285 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:11:42] ENGINE Serving on http://:::9283 2024-04-27T02:11:42.285 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:11:42] ENGINE Bus STARTED 2024-04-27T02:11:42.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:42.259+0000 7f39d5c77000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:11:42.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:42.362+0000 7f39d5c77000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:11:42.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 180' 2024-04-27T02:11:42.766 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:42.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:11:42.766 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:42.698+0000 7f39d5c77000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:11:42.766 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:42.787+0000 7f39d5c77000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: mgrmap e23: x(active, starting, since 0.466485s) 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:11:43.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: Manager daemon x is now available 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: Queued rgw.foo for migration 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: mgrmap e23: x(active, starting, since 0.466485s) 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:11:43.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: Manager daemon x is now available 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: Queued rgw.foo for migration 2024-04-27T02:11:43.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: Deploying cephadm binary to smithi026 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: Deploying cephadm binary to smithi151 2024-04-27T02:11:43.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:43.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: Deploying cephadm binary to smithi026 2024-04-27T02:11:43.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: Deploying cephadm binary to smithi151 2024-04-27T02:11:43.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:42 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: mgrmap e23: x(active, starting, since 0.466485s) 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:11:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: Manager daemon x is now available 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: Queued rgw.foo for migration 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:11:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:11:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: Deploying cephadm binary to smithi026 2024-04-27T02:11:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: Deploying cephadm binary to smithi151 2024-04-27T02:11:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:42 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:43.856 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:11:43.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:11:43.856 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:11:43.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:43.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:11:44.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[21867]: [27/Apr/2024:02:11:42] ENGINE Bus STARTING 2024-04-27T02:11:44.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[21867]: [27/Apr/2024:02:11:42] ENGINE Serving on https://172.21.15.151:7150 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[21867]: mgrmap e24: x(active, since 1.48156s) 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[21867]: [27/Apr/2024:02:11:43] ENGINE Serving on http://172.21.15.151:8765 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[21867]: [27/Apr/2024:02:11:43] ENGINE Bus STARTED 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[26568]: [27/Apr/2024:02:11:42] ENGINE Bus STARTING 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[26568]: [27/Apr/2024:02:11:42] ENGINE Serving on https://172.21.15.151:7150 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[26568]: mgrmap e24: x(active, since 1.48156s) 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:44.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[26568]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:44.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[26568]: [27/Apr/2024:02:11:43] ENGINE Serving on http://172.21.15.151:8765 2024-04-27T02:11:44.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:43 smithi026 ceph-mon[26568]: [27/Apr/2024:02:11:43] ENGINE Bus STARTED 2024-04-27T02:11:44.160 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:44.005+0000 7f39d5c77000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:11:44.160 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:44.180+0000 7f39d5c77000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:11:44.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:43 smithi151 ceph-mon[22663]: [27/Apr/2024:02:11:42] ENGINE Bus STARTING 2024-04-27T02:11:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:43 smithi151 ceph-mon[22663]: [27/Apr/2024:02:11:42] ENGINE Serving on https://172.21.15.151:7150 2024-04-27T02:11:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:43 smithi151 ceph-mon[22663]: mgrmap e24: x(active, since 1.48156s) 2024-04-27T02:11:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:43 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:43 smithi151 ceph-mon[22663]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:43 smithi151 ceph-mon[22663]: [27/Apr/2024:02:11:43] ENGINE Serving on http://172.21.15.151:8765 2024-04-27T02:11:44.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:43 smithi151 ceph-mon[22663]: [27/Apr/2024:02:11:43] ENGINE Bus STARTED 2024-04-27T02:11:44.430 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:44.275+0000 7f39d5c77000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:11:44.678 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:44.605+0000 7f39d5c77000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:11:44.678 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:44.699+0000 7f39d5c77000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:11:44.938 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:44.797+0000 7f39d5c77000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:11:45.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:44 smithi151 ceph-mon[22663]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:45.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:45.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:44 smithi026 ceph-mon[26568]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:45.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:45 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:45.050+0000 7f39d5c77000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:11:45.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:45 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:45.514+0000 7f39d5c77000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:11:46.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:45 smithi151 ceph-mon[22663]: mgrmap e25: x(active, since 3s) 2024-04-27T02:11:46.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:45 smithi026 ceph-mon[21867]: mgrmap e25: x(active, since 3s) 2024-04-27T02:11:46.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:45 smithi026 ceph-mon[26568]: mgrmap e25: x(active, since 3s) 2024-04-27T02:11:46.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:46.022+0000 7f39d5c77000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:11:46.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:46.120+0000 7f39d5c77000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:11:46.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:46.429+0000 7f39d5c77000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:11:46.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:46.524+0000 7f39d5c77000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:11:47.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:46 smithi026 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:47.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:46 smithi026 ceph-mon[26568]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:47.286 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:46.946+0000 7f39d5c77000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:11:47.286 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:47.086+0000 7f39d5c77000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:11:47.286 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:47.183+0000 7f39d5c77000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:11:47.286 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:47.307+0000 7f39d5c77000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:11:47.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:46 smithi151 ceph-mon[22663]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:47.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:46] "GET /metrics HTTP/1.1" 200 34896 "" "Prometheus/2.33.4" 2024-04-27T02:11:47.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:47.406+0000 7f39d5c77000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:11:48.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:48.014+0000 7f39d5c77000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:11:48.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:48.398+0000 7f39d5c77000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:11:48.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:11:48] ENGINE Bus STARTING 2024-04-27T02:11:48.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: CherryPy Checker: 2024-04-27T02:11:48.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: The Application mounted at '' has an empty config. 2024-04-27T02:11:48.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: 2024-04-27T02:11:48.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:11:48] ENGINE Serving on http://:::9283 2024-04-27T02:11:48.857 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: [27/Apr/2024:02:11:48] ENGINE Bus STARTED 2024-04-27T02:11:49.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:48 smithi151 ceph-mon[22663]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:49.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:48 smithi151 ceph-mon[22663]: Standby manager daemon y started 2024-04-27T02:11:49.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:48 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:11:49.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:48 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:11:49.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:48 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:11:49.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:48 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:11:49.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[26568]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[26568]: Standby manager daemon y started 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[21867]: Standby manager daemon y started 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:11:49.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:11:49.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:48 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/5295487' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:11:50.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:49 smithi026 ceph-mon[26568]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-27T02:11:50.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:49 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:11:50.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:49 smithi026 ceph-mon[21867]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-27T02:11:50.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:49 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:11:50.224 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:49 smithi151 ceph-mon[22663]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-27T02:11:50.224 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:49 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:11:51.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:50 smithi026 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:11:51.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:50 smithi026 ceph-mon[26568]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:11:51.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:50 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:50.940Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:51.180 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:50 smithi151 ceph-mon[22663]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:11:52.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:51 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:51 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:51 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:51 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:52.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:51 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-27T02:11:53.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi026 to 2993M 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi026 to 2993M 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:11:53.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi026 to 2993M 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-27T02:11:53.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:53.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:11:53.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:53 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:11:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=error ts=2024-04-27T02:11:53.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.151:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.151 because it doesn't contain any IP SANs" 2024-04-27T02:11:53.857 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:11:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=warn ts=2024-04-27T02:11:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-27T02:11:54.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:11:54.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 bash[55738]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-27T02:11:54.433 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 podman[55751]: 2024-04-27 02:11:54.280974786 +0000 UTC m=+0.099486046 container died 5405a3c0085a28129ef782a9e79ffc6994e52e866163c8ee19ed307a226ceda3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:11:54.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Adjusting osd_memory_target on smithi151 to 3248M 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.751 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.751 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.751 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.751 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.751 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.751 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:54 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.856 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 podman[55751]: 2024-04-27 02:11:54.636994751 +0000 UTC m=+0.455506007 container cleanup 5405a3c0085a28129ef782a9e79ffc6994e52e866163c8ee19ed307a226ceda3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:11:54.856 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 bash[55751]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a 2024-04-27T02:11:54.857 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 bash[55778]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Adjusting osd_memory_target on smithi151 to 3248M 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:11:54.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Adjusting osd_memory_target on smithi151 to 3248M 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:54.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:54 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:55.150 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 podman[55765]: 2024-04-27 02:11:54.914270028 +0000 UTC m=+0.637183590 container remove 5405a3c0085a28129ef782a9e79ffc6994e52e866163c8ee19ed307a226ceda3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:11:55.150 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:11:55.150 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-27T02:11:55.150 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:11:55.151 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.889s CPU time. 2024-04-27T02:11:55.151 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:54 smithi026 systemd[1]: Starting Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:11:55.151 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 podman[55846]: 2024-04-27 02:11:55.056209521 +0000 UTC m=+0.018524712 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-27T02:11:55.151 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 podman[55846]: 2024-04-27 02:11:55.172639303 +0000 UTC m=+0.134954483 container create b2e202622fa12e190495997595c9c294d2c295ff789b9264f6587197d4f392f3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 podman[55846]: 2024-04-27 02:11:55.366679872 +0000 UTC m=+0.328995049 container init b2e202622fa12e190495997595c9c294d2c295ff789b9264f6587197d4f392f3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 podman[55846]: 2024-04-27 02:11:55.370149147 +0000 UTC m=+0.332464325 container start b2e202622fa12e190495997595c9c294d2c295ff789b9264f6587197d4f392f3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.374Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.374Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-27T02:11:55.428 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=arp 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-27T02:11:55.429 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=edac 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-27T02:11:55.430 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=os 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=stat 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-27T02:11:55.431 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=time 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=uname 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.375Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[55860]: ts=2024-04-27T02:11:55.376Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-27T02:11:55.432 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 bash[55846]: b2e202622fa12e190495997595c9c294d2c295ff789b9264f6587197d4f392f3 2024-04-27T02:11:55.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:55 smithi151 ceph-mon[22663]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-27T02:11:55.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:55 smithi151 ceph-mon[22663]: Reconfiguring daemon node-exporter.a on smithi026 2024-04-27T02:11:55.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:55 smithi151 ceph-mon[22663]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:11:55.856 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:11:55 smithi026 systemd[1]: Started Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:11:55.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-mon[21867]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-27T02:11:55.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-mon[21867]: Reconfiguring daemon node-exporter.a on smithi026 2024-04-27T02:11:55.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:11:55.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-mon[26568]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-27T02:11:55.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-mon[26568]: Reconfiguring daemon node-exporter.a on smithi026 2024-04-27T02:11:55.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:55 smithi026 ceph-mon[26568]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:11:56.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:56 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:56.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:56 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:56.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:56 smithi151 ceph-mon[22663]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-27T02:11:56.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:56 smithi151 ceph-mon[22663]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-27T02:11:56.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:56.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:56.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[21867]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-27T02:11:56.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[21867]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-27T02:11:56.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:56.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:11:56.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[26568]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-27T02:11:56.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-mon[26568]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-27T02:11:57.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:11:56 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:56] "GET /metrics HTTP/1.1" 200 34896 "" "Prometheus/2.33.4" 2024-04-27T02:11:57.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:11:56 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:11:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:11:57.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:57 smithi151 ceph-mon[22663]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:11:57.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:57 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:11:57.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:57 smithi026 ceph-mon[26568]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:11:57.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:57 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:11:57.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:57 smithi026 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:11:57.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:57 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:11:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:11:58 smithi151 ceph-mon[22663]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:11:59.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:11:58 smithi026 ceph-mon[26568]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:11:59.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:11:58 smithi026 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:12:01.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:01 smithi026 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:12:01.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:01 smithi026 ceph-mon[26568]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:12:01.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:01 smithi151 ceph-mon[22663]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:12:02.597 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:02 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:02.597 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:02 smithi026 bash[56112]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-27T02:12:02.597 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[46582]: level=info ts=2024-04-27T02:12:02.577Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:12:02.597 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.26:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.151:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:12:02.856 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:02 smithi026 podman[56125]: 2024-04-27 02:12:02.618370182 +0000 UTC m=+0.159589173 container died 5cec00038ac8404a63763b4edba746d3a8f99567837eb85b8d1af52743b90dd5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:03.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:03 smithi026 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:03.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:03 smithi026 ceph-mon[26568]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:03.258 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:02 smithi026 podman[56125]: 2024-04-27 02:12:02.969322444 +0000 UTC m=+0.510541446 container cleanup 5cec00038ac8404a63763b4edba746d3a8f99567837eb85b8d1af52743b90dd5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:03.258 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:02 smithi026 bash[56125]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a 2024-04-27T02:12:03.258 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:02 smithi026 bash[56152]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-27T02:12:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:03 smithi151 ceph-mon[22663]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:03.566 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 podman[56139]: 2024-04-27 02:12:03.279430005 +0000 UTC m=+0.660984807 container remove 5cec00038ac8404a63763b4edba746d3a8f99567837eb85b8d1af52743b90dd5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:03.567 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 podman[56139]: 2024-04-27 02:12:03.371635281 +0000 UTC m=+0.753190081 volume remove 22774dc03a585c4b173fe1fe70815e7cf43a6565b54b969a48b52502ac67ba03 2024-04-27T02:12:03.567 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-27T02:12:03.567 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:03.567 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.049s CPU time. 2024-04-27T02:12:03.567 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 systemd[1]: Starting Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:03.847 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 podman[56220]: 2024-04-27 02:12:03.589154818 +0000 UTC m=+0.087857704 volume create 698372093a792565b93992c8cff7a1ffcd48666ef97f5148511c83ba5c3249a8 2024-04-27T02:12:03.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 podman[56220]: 2024-04-27 02:12:03.520230857 +0000 UTC m=+0.018933755 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-27T02:12:03.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 podman[56220]: 2024-04-27 02:12:03.673006941 +0000 UTC m=+0.171709865 container create 1a93078168a034f1e29fcfdd4acd97e1119ae5e3af9eebcc3d5f3e817ebdcbee (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:04.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:04 smithi026 ceph-mon[21867]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:04.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:04 smithi026 ceph-mon[26568]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 podman[56220]: 2024-04-27 02:12:03.975650738 +0000 UTC m=+0.474353624 container init 1a93078168a034f1e29fcfdd4acd97e1119ae5e3af9eebcc3d5f3e817ebdcbee (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 podman[56220]: 2024-04-27 02:12:03.97852291 +0000 UTC m=+0.477225802 container start 1a93078168a034f1e29fcfdd4acd97e1119ae5e3af9eebcc3d5f3e817ebdcbee (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:03.988Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:03.989Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:03.989Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:03.991Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:04.017Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:04.018Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:12:04.107 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:04.020Z caller=main.go:518 msg=Listening address=:9093 2024-04-27T02:12:04.108 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:04.020Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-27T02:12:04.108 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:04 smithi026 bash[56220]: 1a93078168a034f1e29fcfdd4acd97e1119ae5e3af9eebcc3d5f3e817ebdcbee 2024-04-27T02:12:04.108 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:04 smithi026 systemd[1]: Started Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:04.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:04 smithi151 ceph-mon[22663]: from='client.14805 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:05.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:05 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:05.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:05.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:05.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:12:05.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:05.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:05.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:05.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:12:06.239 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:05 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:05.992Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000720179s 2024-04-27T02:12:07.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:07 smithi026 ceph-mon[21867]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:07.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:07 smithi026 ceph-mon[26568]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:07.202 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:12:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:12:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:12:07.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-mon[22663]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:07.377 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:12:06] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-27T02:12:07.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:12:07.534+0000 7f80bd33e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:07.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:12:07.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:07.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:07.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:07.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:07.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:07.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:07.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:07.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:12:07.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:07.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:07.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:07.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return future.result(timeout) 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.__get_result() 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: raise self._exception 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:07.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:07.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:07.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:07.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:07.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:07.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:08 smithi151 ceph-mon[22663]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:09.348 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:09.348 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:09.348 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:09.348 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:09.348 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:09.349 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:09.350 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: return future.result(timeout) 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.__get_result() 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: raise self._exception 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:09.351 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:09.352 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:08 smithi151 ceph-mon[22663]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:08 smithi151 ceph-mon[22663]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:12:09.353 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:08 smithi151 ceph-mon[22663]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:09.357 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:09.359 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:09.360 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:12:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[21867]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[26568]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:09.362 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:09.363 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:09.364 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:09.365 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[26568]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[26568]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:12:09.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:08 smithi026 ceph-mon[26568]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:11.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:11 smithi151 ceph-mon[22663]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:11.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:11 smithi026 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:11.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:11 smithi026 ceph-mon[26568]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:12.598 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.26:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.151:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:12:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:13 smithi151 ceph-mon[22663]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:12:13.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:13 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:13.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:13 smithi026 ceph-mon[21867]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:12:13.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:13 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:13 smithi026 ceph-mon[26568]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:12:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:13 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:14.356 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:12:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:12:13.994Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003420392s 2024-04-27T02:12:15.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:15 smithi026 ceph-mon[21867]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:12:15.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:15 smithi026 ceph-mon[26568]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:12:15.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:15 smithi151 ceph-mon[22663]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:12:17.010 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 systemd[1]: Stopping Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:17.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-mon[22663]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:17.010 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:12:16] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 bash[46047]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.147Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.148Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-27T02:12:17.348 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.148Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-27T02:12:17.349 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[39600]: ts=2024-04-27T02:12:17.148Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-27T02:12:17.349 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 podman[46059]: 2024-04-27 02:12:17.157784653 +0000 UTC m=+0.116641278 container died 6c19ce81f41e80181eeadcc8761eca75b2af45ed28e142fb31eef540669f0e3b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:17 smithi026 ceph-mon[21867]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:17.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:17 smithi026 ceph-mon[26568]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:17.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:12:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[22096]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:12:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:12:17.768 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 podman[46059]: 2024-04-27 02:12:17.506266905 +0000 UTC m=+0.465123564 container cleanup 6c19ce81f41e80181eeadcc8761eca75b2af45ed28e142fb31eef540669f0e3b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:17.769 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 bash[46059]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a 2024-04-27T02:12:17.769 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 bash[46086]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:12:18.020 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 podman[46073]: 2024-04-27 02:12:17.791260991 +0000 UTC m=+0.630937480 container remove 6c19ce81f41e80181eeadcc8761eca75b2af45ed28e142fb31eef540669f0e3b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:18.020 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-27T02:12:18.020 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 systemd[1]: Stopped Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:18.021 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.982s CPU time. 2024-04-27T02:12:18.021 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:17 smithi151 systemd[1]: Starting Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:18.021 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 podman[46156]: 2024-04-27 02:12:17.92560556 +0000 UTC m=+0.019495885 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-27T02:12:18.021 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 podman[46156]: 2024-04-27 02:12:18.043057434 +0000 UTC m=+0.136947743 container create fcc967e841465452eb134c9881d72362c55d714431559b1713621018f6ac217c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:18.298 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 podman[46156]: 2024-04-27 02:12:18.245664939 +0000 UTC m=+0.339555249 container init fcc967e841465452eb134c9881d72362c55d714431559b1713621018f6ac217c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:18.298 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 podman[46156]: 2024-04-27 02:12:18.248141758 +0000 UTC m=+0.342032078 container start fcc967e841465452eb134c9881d72362c55d714431559b1713621018f6ac217c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:12:18.298 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.272Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-27T02:12:18.298 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.272Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.272Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.272Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi151 (none))" 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.272Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.272Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.273Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.274Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.274Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.277Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.277Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.038µs 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.277Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-27T02:12:18.299 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 bash[46156]: fcc967e841465452eb134c9881d72362c55d714431559b1713621018f6ac217c 2024-04-27T02:12:18.597 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 systemd[1]: Started Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:18.597 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.486Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-27T02:12:19.056 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:18 smithi151 systemd[1]: Stopping Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:19.056 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:18 smithi151 bash[46280]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.709Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.709Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.709Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.139µs wal_replay_duration=432.59862ms total_replay_duration=432.64169ms 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.714Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.714Z caller=main.go:947 level=info msg="TSDB started" 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.714Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.740Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.840887ms db_storage=878ns remote_storage=1.738µs web_handler=454ns query_engine=856ns scrape=719.996µs scrape_sd=158.802µs notify=18.921µs notify_sd=21.048µs rules=24.534249ms 2024-04-27T02:12:19.057 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:18.740Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-27T02:12:19.331 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 podman[46292]: 2024-04-27 02:12:19.078877336 +0000 UTC m=+0.138118933 container died 8e60a39d89eda768cc4fc450eb444789e324886558dda736bc23eef9be23060d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:12:19.597 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 podman[46292]: 2024-04-27 02:12:19.4695242 +0000 UTC m=+0.528765804 container cleanup 8e60a39d89eda768cc4fc450eb444789e324886558dda736bc23eef9be23060d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:12:19.597 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 bash[46292]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b 2024-04-27T02:12:19.598 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 bash[46320]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-27T02:12:19.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:19 smithi151 ceph-mon[22663]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:19.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:19 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:19.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:19 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:19.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:19 smithi151 ceph-mon[22663]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-27T02:12:19.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:19 smithi151 ceph-mon[22663]: Reconfiguring daemon node-exporter.b on smithi151 2024-04-27T02:12:19.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:19 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3991332150' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:12:19.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[21867]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:19.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:19.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[21867]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[21867]: Reconfiguring daemon node-exporter.b on smithi151 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3991332150' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[26568]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[26568]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[26568]: Reconfiguring daemon node-exporter.b on smithi151 2024-04-27T02:12:19.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:19 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3991332150' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:12:19.990 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 podman[46306]: 2024-04-27 02:12:19.77133197 +0000 UTC m=+0.696922071 container remove 8e60a39d89eda768cc4fc450eb444789e324886558dda736bc23eef9be23060d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:12:19.991 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:12:19.991 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-27T02:12:19.991 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 systemd[1]: Stopped Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:19.991 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.825s CPU time. 2024-04-27T02:12:19.991 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:19 smithi151 systemd[1]: Starting Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 podman[46399]: 2024-04-27 02:12:19.915155403 +0000 UTC m=+0.018482054 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 podman[46399]: 2024-04-27 02:12:20.047671988 +0000 UTC m=+0.150998626 container create 00d8989305ef3b78e897401f2fd8bea63c20cd9baeb5f5ddb6e6ccedb66caf5e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 podman[46399]: 2024-04-27 02:12:20.250173514 +0000 UTC m=+0.353500152 container init 00d8989305ef3b78e897401f2fd8bea63c20cd9baeb5f5ddb6e6ccedb66caf5e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 podman[46399]: 2024-04-27 02:12:20.2533127 +0000 UTC m=+0.356639347 container start 00d8989305ef3b78e897401f2fd8bea63c20cd9baeb5f5ddb6e6ccedb66caf5e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.256Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.256Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=arp 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-27T02:12:20.312 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=edac 2024-04-27T02:12:20.313 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-27T02:12:20.314 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=os 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=stat 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-27T02:12:20.315 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=time 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=uname 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.257Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[46414]: ts=2024-04-27T02:12:20.258Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-27T02:12:20.316 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 bash[46399]: 00d8989305ef3b78e897401f2fd8bea63c20cd9baeb5f5ddb6e6ccedb66caf5e 2024-04-27T02:12:20.597 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:12:20 smithi151 systemd[1]: Started Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:20.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3858419261' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]: dispatch 2024-04-27T02:12:20.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:20 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]: dispatch 2024-04-27T02:12:20.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:20 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3858419261' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]: dispatch 2024-04-27T02:12:20.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:20 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]: dispatch 2024-04-27T02:12:20.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:20 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3858419261' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]: dispatch 2024-04-27T02:12:20.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:20 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]: dispatch 2024-04-27T02:12:21.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]': finished 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: osdmap e95: 8 total, 8 up, 8 in 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: Reconfiguring grafana.a (dependencies changed)... 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/212757828' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]: dispatch 2024-04-27T02:12:21.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:21 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]: dispatch 2024-04-27T02:12:21.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:21.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]': finished 2024-04-27T02:12:21.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: osdmap e95: 8 total, 8 up, 8 in 2024-04-27T02:12:21.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: Reconfiguring grafana.a (dependencies changed)... 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/212757828' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]: dispatch 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]: dispatch 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/438351353"}]': finished 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: osdmap e95: 8 total, 8 up, 8 in 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: Reconfiguring grafana.a (dependencies changed)... 2024-04-27T02:12:21.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-27T02:12:21.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:21.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:12:21.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/212757828' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]: dispatch 2024-04-27T02:12:21.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:21 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]: dispatch 2024-04-27T02:12:22.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:22 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:12:22.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:22 smithi151 ceph-mon[22663]: Reconfiguring daemon grafana.a on smithi151 2024-04-27T02:12:22.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:22 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]': finished 2024-04-27T02:12:22.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:22 smithi151 ceph-mon[22663]: osdmap e96: 8 total, 8 up, 8 in 2024-04-27T02:12:22.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:22 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1960938560' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/570608856"}]: dispatch 2024-04-27T02:12:22.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:12:22.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[21867]: Reconfiguring daemon grafana.a on smithi151 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]': finished 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[21867]: osdmap e96: 8 total, 8 up, 8 in 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1960938560' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/570608856"}]: dispatch 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[26568]: Reconfiguring daemon grafana.a on smithi151 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/570608856"}]': finished 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[26568]: osdmap e96: 8 total, 8 up, 8 in 2024-04-27T02:12:22.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:22 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1960938560' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/570608856"}]: dispatch 2024-04-27T02:12:23.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:23 smithi151 ceph-mon[22663]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:12:23.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:23 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1960938560' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/570608856"}]': finished 2024-04-27T02:12:23.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:23 smithi151 ceph-mon[22663]: osdmap e97: 8 total, 8 up, 8 in 2024-04-27T02:12:23.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:23 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2915217844' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/820053206"}]: dispatch 2024-04-27T02:12:23.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[21867]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:12:23.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1960938560' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/570608856"}]': finished 2024-04-27T02:12:23.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[21867]: osdmap e97: 8 total, 8 up, 8 in 2024-04-27T02:12:23.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2915217844' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/820053206"}]: dispatch 2024-04-27T02:12:23.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[26568]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:12:23.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1960938560' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/570608856"}]': finished 2024-04-27T02:12:23.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[26568]: osdmap e97: 8 total, 8 up, 8 in 2024-04-27T02:12:23.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:23 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2915217844' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/820053206"}]: dispatch 2024-04-27T02:12:24.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:12:23] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-04-27T02:12:25.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2915217844' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/820053206"}]': finished 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[21867]: osdmap e98: 8 total, 8 up, 8 in 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[21867]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4236174881' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]: dispatch 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]: dispatch 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2915217844' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/820053206"}]': finished 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[26568]: osdmap e98: 8 total, 8 up, 8 in 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[26568]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4236174881' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]: dispatch 2024-04-27T02:12:25.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:24 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]: dispatch 2024-04-27T02:12:25.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:24 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2915217844' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/820053206"}]': finished 2024-04-27T02:12:25.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:24 smithi151 ceph-mon[22663]: osdmap e98: 8 total, 8 up, 8 in 2024-04-27T02:12:25.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:24 smithi151 ceph-mon[22663]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:25.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:24 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4236174881' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]: dispatch 2024-04-27T02:12:25.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:24 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]: dispatch 2024-04-27T02:12:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:25 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]': finished 2024-04-27T02:12:26.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:25 smithi151 ceph-mon[22663]: osdmap e99: 8 total, 8 up, 8 in 2024-04-27T02:12:26.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:25 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1649847437' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2781242146"}]: dispatch 2024-04-27T02:12:26.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:25 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]': finished 2024-04-27T02:12:26.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:25 smithi026 ceph-mon[26568]: osdmap e99: 8 total, 8 up, 8 in 2024-04-27T02:12:26.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:25 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1649847437' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2781242146"}]: dispatch 2024-04-27T02:12:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:25 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2500712321"}]': finished 2024-04-27T02:12:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:25 smithi026 ceph-mon[21867]: osdmap e99: 8 total, 8 up, 8 in 2024-04-27T02:12:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:25 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1649847437' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2781242146"}]: dispatch 2024-04-27T02:12:27.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:26 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1649847437' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2781242146"}]': finished 2024-04-27T02:12:27.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:26 smithi151 ceph-mon[22663]: osdmap e100: 8 total, 8 up, 8 in 2024-04-27T02:12:27.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:26 smithi151 ceph-mon[22663]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:27.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:26 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3923847491' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]: dispatch 2024-04-27T02:12:27.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:26 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]: dispatch 2024-04-27T02:12:27.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1649847437' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2781242146"}]': finished 2024-04-27T02:12:27.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[26568]: osdmap e100: 8 total, 8 up, 8 in 2024-04-27T02:12:27.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[26568]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:27.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3923847491' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]: dispatch 2024-04-27T02:12:27.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]: dispatch 2024-04-27T02:12:27.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1649847437' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2781242146"}]': finished 2024-04-27T02:12:27.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[21867]: osdmap e100: 8 total, 8 up, 8 in 2024-04-27T02:12:27.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:27.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3923847491' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]: dispatch 2024-04-27T02:12:27.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:26 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]: dispatch 2024-04-27T02:12:28.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:27 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]': finished 2024-04-27T02:12:28.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:27 smithi151 ceph-mon[22663]: osdmap e101: 8 total, 8 up, 8 in 2024-04-27T02:12:28.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:27 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:28.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:27 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]': finished 2024-04-27T02:12:28.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:27 smithi026 ceph-mon[26568]: osdmap e101: 8 total, 8 up, 8 in 2024-04-27T02:12:28.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:27 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:28.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:27 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1449023003"}]': finished 2024-04-27T02:12:28.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:27 smithi026 ceph-mon[21867]: osdmap e101: 8 total, 8 up, 8 in 2024-04-27T02:12:28.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:27 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:28.894 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:12:29.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:28 smithi151 ceph-mon[22663]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:12:29.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:28 smithi026 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:12:29.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:28 smithi026 ceph-mon[26568]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:12:30.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:29 smithi151 ceph-mon[22663]: from='client.24863 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:30.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:29 smithi026 ceph-mon[21867]: from='client.24863 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:29 smithi026 ceph-mon[26568]: from='client.24863 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:31.052 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:30 smithi151 ceph-mon[22663]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:31.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:30 smithi026 ceph-mon[26568]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:30 smithi026 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:12:32.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:31 smithi151 systemd[1]: Stopping Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:32.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:31 smithi151 bash[46698]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-27T02:12:32.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[38798]: t=2024-04-27T02:12:31+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-27T02:12:32.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 podman[46711]: 2024-04-27 02:12:32.008741906 +0000 UTC m=+0.139986210 container died 611ca410ba2628897feb74396608b8b0d3b582b2c308ad67fb512ae33e523c02 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , name=ubi8, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, vcs-type=git, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-03-28T10:36:18.413762) 2024-04-27T02:12:32.597 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 podman[46711]: 2024-04-27 02:12:32.365905855 +0000 UTC m=+0.497150156 container cleanup 611ca410ba2628897feb74396608b8b0d3b582b2c308ad67fb512ae33e523c02 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, maintainer=Paul Cuzner , architecture=x86_64, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, release=236.1648460182, vcs-type=git, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-27T02:12:32.597 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 bash[46711]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a 2024-04-27T02:12:32.598 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 bash[46738]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-27T02:12:32.863 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 podman[46726]: 2024-04-27 02:12:32.634279224 +0000 UTC m=+0.622093865 container remove 611ca410ba2628897feb74396608b8b0d3b582b2c308ad67fb512ae33e523c02 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, io.openshift.expose-services=, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, version=8.5, vcs-type=git, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner ) 2024-04-27T02:12:32.863 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-27T02:12:32.863 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 systemd[1]: Stopped Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:32.863 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Consumed 3.398s CPU time. 2024-04-27T02:12:32.863 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 systemd[1]: Starting Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:32.863 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 podman[46811]: 2024-04-27 02:12:32.776807025 +0000 UTC m=+0.017018736 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-27T02:12:32.863 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:32 smithi151 podman[46811]: 2024-04-27 02:12:32.885602707 +0000 UTC m=+0.125814418 container create 86da930ab0011b48ac58e122c45be2b8aec5f67bf2c4a4966f0ed50e0fbeb35f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, io.openshift.tags=base rhel8, version=8.5, architecture=x86_64, release=236.1648460182, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-27T02:12:33.130 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-mon[22663]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 724 B/s rd, 0 op/s 2024-04-27T02:12:33.130 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 podman[46811]: 2024-04-27 02:12:33.122567059 +0000 UTC m=+0.362778770 container init 86da930ab0011b48ac58e122c45be2b8aec5f67bf2c4a4966f0ed50e0fbeb35f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, maintainer=Paul Cuzner , io.openshift.expose-services=, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, name=ubi8, release=236.1648460182, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.buildah.version=1.24.2, description=Ceph Grafana Container) 2024-04-27T02:12:33.130 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 podman[46811]: 2024-04-27 02:12:33.125135204 +0000 UTC m=+0.365346924 container start 86da930ab0011b48ac58e122c45be2b8aec5f67bf2c4a4966f0ed50e0fbeb35f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, release=236.1648460182, vendor=Red Hat, Inc., io.openshift.expose-services=, name=ubi8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-27T02:12:33.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:33 smithi026 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 724 B/s rd, 0 op/s 2024-04-27T02:12:33.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:33 smithi026 ceph-mon[26568]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 724 B/s rd, 0 op/s 2024-04-27T02:12:33.390 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STOPPING 2024-04-27T02:12:33.390 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:12:33.390 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STOPPED 2024-04-27T02:12:33.390 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STARTING 2024-04-27T02:12:33.390 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Serving on http://:::9283 2024-04-27T02:12:33.390 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STARTED 2024-04-27T02:12:33.390 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[46825]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:12:33.390 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[46825]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:12:33.390 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 bash[46811]: 86da930ab0011b48ac58e122c45be2b8aec5f67bf2c4a4966f0ed50e0fbeb35f 2024-04-27T02:12:33.391 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 systemd[1]: Started Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:33.391 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 podman[46841]: 2024-04-27 02:12:33.234538479 +0000 UTC m=+0.072506935 container died 86da930ab0011b48ac58e122c45be2b8aec5f67bf2c4a4966f0ed50e0fbeb35f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, name=ubi8, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.openshift.expose-services=, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git) 2024-04-27T02:12:33.788 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STOPPING 2024-04-27T02:12:33.788 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:12:33.788 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STOPPED 2024-04-27T02:12:33.788 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STARTING 2024-04-27T02:12:33.788 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Serving on http://:::9283 2024-04-27T02:12:33.789 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STARTED 2024-04-27T02:12:33.789 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STOPPING 2024-04-27T02:12:34.074 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 podman[46841]: 2024-04-27 02:12:33.810699208 +0000 UTC m=+0.648667643 container remove 86da930ab0011b48ac58e122c45be2b8aec5f67bf2c4a4966f0ed50e0fbeb35f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.expose-services=, version=8.5, io.buildah.version=1.24.2, distribution-scope=public, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-27T02:12:34.074 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-27T02:12:34.074 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:33 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-27T02:12:34.075 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:12:34.075 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STOPPED 2024-04-27T02:12:34.075 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:33] ENGINE Bus STARTING 2024-04-27T02:12:34.075 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:34] ENGINE Serving on http://:::9283 2024-04-27T02:12:34.075 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:12:34] ENGINE Bus STARTED 2024-04-27T02:12:34.330 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:12:34.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:12:34.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2024-04-27T02:12:34.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2024-04-27T02:12:34.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:34 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:12:34.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:12:34.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:12:34.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:34.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:34 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:12:35.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:35 smithi026 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:12:35.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:35 smithi026 ceph-mon[26568]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:12:35.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:35 smithi151 ceph-mon[22663]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:12:37.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-27T02:12:37.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:37.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:12:37.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:37 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-27T02:12:37.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:37.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:12:37.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:37 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:38.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:38 smithi151 ceph-mon[22663]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-27T02:12:38.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:38 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:12:38.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:38 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:38.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:38 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:38.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:38 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:38.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:38 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:12:38.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:38 smithi151 ceph-mon[22663]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:12:38.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-27T02:12:38.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:12:38.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[21867]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[26568]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:12:38.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:38 smithi026 ceph-mon[26568]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:12:40.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:40 smithi026 ceph-mon[21867]: from='client.24863 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:40.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:40 smithi026 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:40.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:40 smithi026 ceph-mon[26568]: from='client.24863 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:40.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:40 smithi026 ceph-mon[26568]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-mon[22663]: from='client.24863 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:12:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-mon[22663]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:12:40.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:12:40.563+0000 7f80bd33e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:40.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:12:40.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:40.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:40.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:40.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:40.598 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:40.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:40.600 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return future.result(timeout) 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.__get_result() 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: raise self._exception 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:40.601 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:40.602 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:42 smithi151 ceph-mon[22663]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:42.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:42.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:42.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:42.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:42.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:42.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: return future.result(timeout) 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.__get_result() 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:42.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: raise self._exception 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:42.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:42.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:42 smithi151 ceph-mon[22663]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 813 B/s rd, 0 op/s 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:42 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:42.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:42.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:42.606 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:42.606 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:42.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:42.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:42.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:42.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[26568]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-27T02:12:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:12:42.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:42.615 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 813 B/s rd, 0 op/s 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[26568]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 813 B/s rd, 0 op/s 2024-04-27T02:12:42.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:12:42.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:44.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:43 smithi151 ceph-mon[22663]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:12:44.118 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:12:43] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-27T02:12:44.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:43 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-27T02:12:44.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:43 smithi151 systemd[1]: Stopped Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:44.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:43 smithi151 systemd[1]: Starting Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:44.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 podman[47483]: 2024-04-27 02:12:44.01831129 +0000 UTC m=+0.018119108 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-27T02:12:44.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 podman[47483]: 2024-04-27 02:12:44.140114104 +0000 UTC m=+0.139921909 container create 5ee691df8613b66faa8baa12e10696ae194726e44c86054f982e2fa8ccf6053b (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , vendor=Red Hat, Inc., architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, release=236.1648460182) 2024-04-27T02:12:44.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:43 smithi026 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:12:44.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:43 smithi026 ceph-mon[26568]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:12:44.384 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 podman[47483]: 2024-04-27 02:12:44.37682019 +0000 UTC m=+0.376627995 container init 5ee691df8613b66faa8baa12e10696ae194726e44c86054f982e2fa8ccf6053b (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, release=236.1648460182, vcs-type=git, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, version=8.5, io.buildah.version=1.24.2, vendor=Red Hat, Inc.) 2024-04-27T02:12:44.385 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 podman[47483]: 2024-04-27 02:12:44.380091408 +0000 UTC m=+0.379899219 container start 5ee691df8613b66faa8baa12e10696ae194726e44c86054f982e2fa8ccf6053b (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., name=ubi8, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, release=236.1648460182, version=8.5, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public) 2024-04-27T02:12:44.847 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47497]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:12:44.847 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47497]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:12:44.847 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 bash[47483]: 5ee691df8613b66faa8baa12e10696ae194726e44c86054f982e2fa8ccf6053b 2024-04-27T02:12:44.847 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 systemd[1]: Started Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:44.848 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:44 smithi151 podman[47514]: 2024-04-27 02:12:44.539893414 +0000 UTC m=+0.122748282 container died 5ee691df8613b66faa8baa12e10696ae194726e44c86054f982e2fa8ccf6053b (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , architecture=x86_64, distribution-scope=public, io.buildah.version=1.24.2, description=Ceph Grafana Container, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-27T02:12:45.347 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:45 smithi151 podman[47514]: 2024-04-27 02:12:45.057072642 +0000 UTC m=+0.639927510 container remove 5ee691df8613b66faa8baa12e10696ae194726e44c86054f982e2fa8ccf6053b (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vendor=Red Hat, Inc., io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Paul Cuzner , version=8.5, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-04-27T02:12:45.347 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:45 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-27T02:12:45.348 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:45 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-27T02:12:46.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:46 smithi151 ceph-mon[22663]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-27T02:12:46.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:46 smithi026 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-27T02:12:46.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:46 smithi026 ceph-mon[26568]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-27T02:12:48.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:48 smithi151 ceph-mon[22663]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-27T02:12:48.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:48 smithi026 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-27T02:12:48.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:48 smithi026 ceph-mon[26568]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-27T02:12:49.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:12:50.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:50 smithi151 ceph-mon[22663]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:12:50.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:50 smithi026 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:12:50.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:50 smithi026 ceph-mon[26568]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:12:52.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:52 smithi151 ceph-mon[22663]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:12:52.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:52 smithi026 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:12:52.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:52 smithi026 ceph-mon[26568]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:12:53.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:53 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1867100556' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:12:53.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:53 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1867100556' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:12:53.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:53 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1867100556' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:12:54.331 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:12:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:12:53] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-27T02:12:54.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:54 smithi151 ceph-mon[22663]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:54.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:54 smithi026 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:54.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:54 smithi026 ceph-mon[26568]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-27T02:12:55.385 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-27T02:12:55.386 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 systemd[1]: Stopped Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:55.386 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 systemd[1]: Starting Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:12:55.386 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 podman[47595]: 2024-04-27 02:12:55.271042129 +0000 UTC m=+0.023871943 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-27T02:12:55.707 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 podman[47595]: 2024-04-27 02:12:55.408304155 +0000 UTC m=+0.161133941 container create 9935bc916463a2b08b72c8b0d37c54e6254bddd62c8aae77b3a4dd5c9656f447 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , release=236.1648460182, vcs-type=git, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-04-27T02:12:55.707 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 podman[47595]: 2024-04-27 02:12:55.62847639 +0000 UTC m=+0.381306189 container init 9935bc916463a2b08b72c8b0d37c54e6254bddd62c8aae77b3a4dd5c9656f447 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, maintainer=Paul Cuzner , release=236.1648460182, name=ubi8, io.openshift.expose-services=, architecture=x86_64) 2024-04-27T02:12:55.707 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 podman[47595]: 2024-04-27 02:12:55.631297324 +0000 UTC m=+0.384127111 container start 9935bc916463a2b08b72c8b0d37c54e6254bddd62c8aae77b3a4dd5c9656f447 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, version=8.5, description=Ceph Grafana Container, release=236.1648460182, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-03-28T10:36:18.413762, distribution-scope=public, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., name=ubi8, io.buildah.version=1.24.2, maintainer=Paul Cuzner ) 2024-04-27T02:12:55.707 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47609]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:12:55.707 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47609]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:12:55.707 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 bash[47595]: 9935bc916463a2b08b72c8b0d37c54e6254bddd62c8aae77b3a4dd5c9656f447 2024-04-27T02:12:55.960 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 systemd[1]: Started Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:12:55.960 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:55 smithi151 podman[47626]: 2024-04-27 02:12:55.737220701 +0000 UTC m=+0.072914637 container died 9935bc916463a2b08b72c8b0d37c54e6254bddd62c8aae77b3a4dd5c9656f447 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.buildah.version=1.24.2, io.openshift.expose-services=, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, vendor=Red Hat, Inc., version=8.5, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-27T02:12:56.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:56 smithi151 ceph-mon[22663]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-27T02:12:56.597 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:56 smithi151 podman[47626]: 2024-04-27 02:12:56.280077569 +0000 UTC m=+0.615771506 container remove 9935bc916463a2b08b72c8b0d37c54e6254bddd62c8aae77b3a4dd5c9656f447 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, com.redhat.component=ubi8-container, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, architecture=x86_64, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner ) 2024-04-27T02:12:56.598 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:56 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-27T02:12:56.598 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:12:56 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-27T02:12:56.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:56 smithi026 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-27T02:12:56.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:56 smithi026 ceph-mon[26568]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-27T02:12:57.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:57 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:57.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:57 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:57.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:57 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:12:58.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:12:58 smithi151 ceph-mon[22663]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-27T02:12:58.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:12:58 smithi026 ceph-mon[26568]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-27T02:12:58.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:12:58 smithi026 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-27T02:12:59.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:12:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:12:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:13:00.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:00 smithi151 ceph-mon[22663]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:00.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:00 smithi026 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:00.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:00 smithi026 ceph-mon[26568]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:02.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:02 smithi151 ceph-mon[22663]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:02.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:02 smithi026 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:02.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:02 smithi026 ceph-mon[26568]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:03.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:03 smithi151 ceph-mon[22663]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:03.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:03 smithi026 ceph-mon[21867]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:03.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:03 smithi026 ceph-mon[26568]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:04.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:13:03] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-27T02:13:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:04 smithi026 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-27T02:13:04.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:04 smithi026 ceph-mon[26568]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-27T02:13:04.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:04 smithi151 ceph-mon[22663]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-27T02:13:06.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:06 smithi026 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:06.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:06 smithi026 ceph-mon[26568]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:06.607 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:06 smithi151 ceph-mon[22663]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:06.607 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-27T02:13:06.608 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 systemd[1]: Stopped Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:13:06.608 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 systemd[1]: Starting Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:13:06.608 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 podman[47695]: 2024-04-27 02:13:06.506375209 +0000 UTC m=+0.017967305 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-27T02:13:06.608 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 podman[47695]: 2024-04-27 02:13:06.630414996 +0000 UTC m=+0.142007078 container create 639c5c1ba60ffc8090e2dd93c507e823639fced24e4e561c02bdf5346d41b21e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, release=236.1648460182, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.openshift.expose-services=, io.buildah.version=1.24.2) 2024-04-27T02:13:06.873 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 podman[47695]: 2024-04-27 02:13:06.866684564 +0000 UTC m=+0.378276647 container init 639c5c1ba60ffc8090e2dd93c507e823639fced24e4e561c02bdf5346d41b21e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=236.1648460182, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , name=ubi8, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64) 2024-04-27T02:13:06.873 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 podman[47695]: 2024-04-27 02:13:06.869971047 +0000 UTC m=+0.381563134 container start 639c5c1ba60ffc8090e2dd93c507e823639fced24e4e561c02bdf5346d41b21e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, distribution-scope=public, io.buildah.version=1.24.2, maintainer=Paul Cuzner , release=236.1648460182, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, version=8.5, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-27T02:13:07.347 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47710]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:13:07.348 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47710]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:13:07.348 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 bash[47695]: 639c5c1ba60ffc8090e2dd93c507e823639fced24e4e561c02bdf5346d41b21e 2024-04-27T02:13:07.348 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 systemd[1]: Started Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:13:07.348 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:06 smithi151 podman[47726]: 2024-04-27 02:13:06.974601742 +0000 UTC m=+0.072082524 container died 639c5c1ba60ffc8090e2dd93c507e823639fced24e4e561c02bdf5346d41b21e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, version=8.5, maintainer=Paul Cuzner , architecture=x86_64) 2024-04-27T02:13:07.847 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:07 smithi151 podman[47726]: 2024-04-27 02:13:07.538250901 +0000 UTC m=+0.635731673 container remove 639c5c1ba60ffc8090e2dd93c507e823639fced24e4e561c02bdf5346d41b21e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.buildah.version=1.24.2, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vcs-type=git, release=236.1648460182, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, name=ubi8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-04-27T02:13:07.847 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:07 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-27T02:13:07.847 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:07 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-27T02:13:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:08 smithi026 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:08.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:08 smithi026 ceph-mon[26568]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:08.636 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:08 smithi151 ceph-mon[22663]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:09.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:13:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:13:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:13:10.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:10 smithi151 ceph-mon[22663]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:10.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:10 smithi026 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:10.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:10 smithi026 ceph-mon[26568]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:12.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:12 smithi151 ceph-mon[22663]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:12.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:12 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:12.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:12 smithi026 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:12.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:12 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:12.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:12 smithi026 ceph-mon[26568]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:12.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:12 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:13.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:13 smithi151 ceph-mon[22663]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:13.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:13 smithi026 ceph-mon[21867]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:13.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:13 smithi026 ceph-mon[26568]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:14.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:13:13] "GET /metrics HTTP/1.1" 200 37962 "" "Prometheus/2.33.4" 2024-04-27T02:13:14.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:14 smithi151 ceph-mon[22663]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:14.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:14 smithi026 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:14.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:14 smithi026 ceph-mon[26568]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:16.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:16 smithi151 ceph-mon[22663]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:16.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:16 smithi026 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:16.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:16 smithi026 ceph-mon[26568]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:17.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:17 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-27T02:13:17.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:17 smithi151 systemd[1]: Stopped Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:13:17.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:17 smithi151 systemd[1]: Starting Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:13:17.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:17 smithi151 podman[47794]: 2024-04-27 02:13:17.753403615 +0000 UTC m=+0.018027575 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-27T02:13:17.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:17 smithi151 podman[47794]: 2024-04-27 02:13:17.897855748 +0000 UTC m=+0.162479692 container create 49ee0b0d9f5e75f8c3f6ac64c3a218de47686c707b42243391e6dd083ba97ed7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, version=8.5, io.buildah.version=1.24.2, maintainer=Paul Cuzner , name=ubi8, release=236.1648460182, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64) 2024-04-27T02:13:18.186 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 podman[47794]: 2024-04-27 02:13:18.100567008 +0000 UTC m=+0.365190963 container init 49ee0b0d9f5e75f8c3f6ac64c3a218de47686c707b42243391e6dd083ba97ed7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.24.2, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, name=ubi8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-27T02:13:18.187 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 podman[47794]: 2024-04-27 02:13:18.103099326 +0000 UTC m=+0.367723275 container start 49ee0b0d9f5e75f8c3f6ac64c3a218de47686c707b42243391e6dd083ba97ed7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., description=Ceph Grafana Container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, architecture=x86_64, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-27T02:13:18.187 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47809]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:13:18.187 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[47809]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-27T02:13:18.482 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:18 smithi151 ceph-mon[22663]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:18.482 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 bash[47794]: 49ee0b0d9f5e75f8c3f6ac64c3a218de47686c707b42243391e6dd083ba97ed7 2024-04-27T02:13:18.482 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 systemd[1]: Started Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:13:18.482 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 podman[47825]: 2024-04-27 02:13:18.232028751 +0000 UTC m=+0.097411941 container died 49ee0b0d9f5e75f8c3f6ac64c3a218de47686c707b42243391e6dd083ba97ed7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , distribution-scope=public, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vcs-type=git, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container) 2024-04-27T02:13:18.741 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:13:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:13:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:13:18.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:18 smithi026 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:18.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:18 smithi026 ceph-mon[26568]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:19.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 podman[47825]: 2024-04-27 02:13:18.76414029 +0000 UTC m=+0.629523517 container remove 49ee0b0d9f5e75f8c3f6ac64c3a218de47686c707b42243391e6dd083ba97ed7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, version=8.5, release=236.1648460182, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-27T02:13:19.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-27T02:13:19.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:18 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-27T02:13:20.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:20 smithi151 ceph-mon[22663]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:20.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:20 smithi026 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:20.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:20 smithi026 ceph-mon[26568]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:22.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:22 smithi151 ceph-mon[22663]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:22.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:22 smithi026 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:22.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:22 smithi026 ceph-mon[26568]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:23.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:23 smithi151 ceph-mon[22663]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:23.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:23 smithi026 ceph-mon[21867]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:23.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:23 smithi026 ceph-mon[26568]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:24.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:13:23] "GET /metrics HTTP/1.1" 200 37962 "" "Prometheus/2.33.4" 2024-04-27T02:13:24.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:24 smithi151 ceph-mon[22663]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:24.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:24 smithi026 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:24.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:24 smithi026 ceph-mon[26568]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:26.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:26 smithi151 ceph-mon[22663]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:26.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:26 smithi026 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:26.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:26 smithi026 ceph-mon[26568]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:27 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:27.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:27 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:27.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:27 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:28.637 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:28 smithi151 ceph-mon[22663]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:28.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:28 smithi026 ceph-mon[26568]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:28.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:28 smithi026 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:29.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:13:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:13:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:13:29.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:28 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-27T02:13:29.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:28 smithi151 systemd[1]: Stopped Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:13:29.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:28 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Start request repeated too quickly. 2024-04-27T02:13:29.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:28 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-27T02:13:29.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:13:28 smithi151 systemd[1]: Failed to start Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:13:30.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:30 smithi151 ceph-mon[22663]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:30.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:30 smithi026 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:30.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:30 smithi026 ceph-mon[26568]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:32.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:32 smithi151 ceph-mon[22663]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:32.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:32 smithi026 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:32.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:32 smithi026 ceph-mon[26568]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:33.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:33 smithi151 ceph-mon[22663]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:33.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:33 smithi026 ceph-mon[21867]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:33.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:33 smithi026 ceph-mon[26568]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:34.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:13:33] "GET /metrics HTTP/1.1" 200 37962 "" "Prometheus/2.33.4" 2024-04-27T02:13:34.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:34 smithi151 ceph-mon[22663]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:34.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:34 smithi026 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:34.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:34 smithi026 ceph-mon[26568]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:36.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:36 smithi151 ceph-mon[22663]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:36.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:36 smithi026 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:36.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:36 smithi026 ceph-mon[26568]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:38.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:38 smithi151 ceph-mon[22663]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:38.848 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:13:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:13:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:13:38.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:38 smithi026 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:38.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:38 smithi026 ceph-mon[26568]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:40.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:40 smithi151 ceph-mon[22663]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:40 smithi026 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:40.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:40 smithi026 ceph-mon[26568]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:41.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:41 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:13:41.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:41 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:13:41.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:41 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:13:42.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:42 smithi151 ceph-mon[22663]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:42.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:42.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:42 smithi026 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:42.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:42.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:42 smithi026 ceph-mon[26568]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:13:42.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:43.773 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:43 smithi151 ceph-mon[22663]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:43.779 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:43 smithi026 ceph-mon[21867]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:43.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:43 smithi026 ceph-mon[26568]: from='client.24923 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:13:44.097 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:13:43] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-27T02:13:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:44 smithi151 ceph-mon[22663]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:44 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:44.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:44 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:44.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:44 smithi026 ceph-mon[26568]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:44.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:44 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:44.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:44 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:44.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:44 smithi026 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:13:44.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:44 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:44.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:44 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:13:45.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:13:45.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-04-27T02:13:45.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:13:45.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-04-27T02:13:45.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:45.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:13:45.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:13:45.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:13:45.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:13:45.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:45 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:13:45.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:13:45.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:13:45.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-04-27T02:13:45.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:45.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:13:45.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:13:45.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:13:45.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:13:45.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:13:45.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:45 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:13:48.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:48 smithi026 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:48.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:48 smithi026 ceph-mon[26568]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:48.537 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-mon[22663]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:48.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:13:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:13:48.559+0000 7f80bd33e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:13:48.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:13:48.849 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return future.result(timeout) 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.__get_result() 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: raise self._exception 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: config_blobs=daemon_spec.final_config, 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:13:48.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:13:48.851 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:48.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:50 smithi151 ceph-mon[22663]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:13:50.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:13:50.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: return future.result(timeout) 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.__get_result() 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: raise self._exception 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:50.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:50.601 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:50.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.602 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:50 smithi151 ceph-mon[22663]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:50.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:50 smithi026 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:50.606 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:13:50.606 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:50.607 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-27T02:13:50.608 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.609 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:50.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:50 smithi026 ceph-mon[26568]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:50.611 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:13:50.612 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:13:50.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:13:50.614 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:50 smithi026 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:13:50.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:50 smithi026 ceph-mon[26568]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:52.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:52 smithi026 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:52.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:52 smithi026 ceph-mon[26568]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:52.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:52 smithi151 ceph-mon[22663]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:13:54.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:53 smithi151 ceph-mon[22663]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 521 B/s rd, 0 op/s 2024-04-27T02:13:54.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:13:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:13:53] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-27T02:13:54.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:53 smithi026 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 521 B/s rd, 0 op/s 2024-04-27T02:13:54.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:53 smithi026 ceph-mon[26568]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 521 B/s rd, 0 op/s 2024-04-27T02:13:56.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:56 smithi151 ceph-mon[22663]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:56.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:56 smithi026 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:56.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:56 smithi026 ceph-mon[26568]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:13:57.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:57 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:57.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:57 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:57.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:57 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:13:58.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:13:58 smithi151 ceph-mon[22663]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:13:58.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:13:58 smithi026 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:13:58.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:13:58 smithi026 ceph-mon[26568]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:13:59.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:13:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:13:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:14:00.465 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:00 smithi026 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:14:00.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:00 smithi026 ceph-mon[26568]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:14:00.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:00 smithi151 ceph-mon[22663]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:14:01.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:01 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/46739271' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:14:01.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:01 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/46739271' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:14:01.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:01 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/46739271' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:14:02.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:02 smithi151 ceph-mon[22663]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:14:02.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:02 smithi026 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:14:02.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:02 smithi026 ceph-mon[26568]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:14:04.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:03 smithi026 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:04.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:03 smithi026 ceph-mon[26568]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:04.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:03 smithi151 ceph-mon[22663]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:04.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:14:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:14:03] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-27T02:14:06.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:06 smithi151 ceph-mon[22663]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:06.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:06 smithi026 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:06.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:06 smithi026 ceph-mon[26568]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:08.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:08 smithi151 ceph-mon[22663]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:08.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:08 smithi026 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:08 smithi026 ceph-mon[26568]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:14:09.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:14:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:14:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:14:10.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:10 smithi151 ceph-mon[22663]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:14:10.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:10 smithi026 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:14:10.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:10 smithi026 ceph-mon[26568]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:14:11.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:11 smithi151 ceph-mon[22663]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:11.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:11 smithi026 ceph-mon[21867]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:11.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:11 smithi026 ceph-mon[26568]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:12.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:12 smithi151 ceph-mon[22663]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:14:12.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:12 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:12.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:12 smithi026 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:14:12.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:12 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:12.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:12 smithi026 ceph-mon[26568]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:14:12.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:12 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:14.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:13 smithi151 ceph-mon[22663]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:14:14.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:14:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:14:13] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-27T02:14:14.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:13 smithi026 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:14:14.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:13 smithi026 ceph-mon[26568]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:14:16.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:16 smithi151 ceph-mon[22663]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:16.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:16 smithi026 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:16.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:16 smithi026 ceph-mon[26568]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:18.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:18 smithi151 ceph-mon[22663]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:18.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:18 smithi026 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:18.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:18 smithi026 ceph-mon[26568]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:19.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:14:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:14:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:14:20.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:20 smithi151 ceph-mon[22663]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:20.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:20 smithi026 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:20.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:20 smithi026 ceph-mon[26568]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:21.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:21 smithi151 ceph-mon[22663]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:21.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:21 smithi026 ceph-mon[21867]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:21.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:21 smithi026 ceph-mon[26568]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:22.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:22 smithi026 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:22.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:22 smithi026 ceph-mon[26568]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:22.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:22 smithi151 ceph-mon[22663]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:24.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:23 smithi151 ceph-mon[22663]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:24.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:14:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:14:23] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-27T02:14:24.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:23 smithi026 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:24.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:23 smithi026 ceph-mon[26568]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:26.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:26 smithi151 ceph-mon[22663]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:26.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:26 smithi026 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:26.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:26 smithi026 ceph-mon[26568]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:27.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:27 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:27.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:27 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:27.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:27 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:28.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:28 smithi151 ceph-mon[22663]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:28.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:28 smithi026 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:28.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:28 smithi026 ceph-mon[26568]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:29.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:14:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:14:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:14:30.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:30 smithi151 ceph-mon[22663]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:30.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:30 smithi026 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:30.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:30 smithi026 ceph-mon[26568]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:31.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:31 smithi151 ceph-mon[22663]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:31.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:31 smithi026 ceph-mon[21867]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:31.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:31 smithi026 ceph-mon[26568]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:32.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:32 smithi151 ceph-mon[22663]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:32.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:32 smithi026 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:32.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:32 smithi026 ceph-mon[26568]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:34.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:33 smithi026 ceph-mon[26568]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:33 smithi026 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:34.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:33 smithi151 ceph-mon[22663]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:34.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:14:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:14:33] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-04-27T02:14:36.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:36 smithi151 ceph-mon[22663]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:36.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:36 smithi026 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:36.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:36 smithi026 ceph-mon[26568]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:38.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:38 smithi151 ceph-mon[22663]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:38.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:38 smithi026 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:38.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:38 smithi026 ceph-mon[26568]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:39.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:14:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:14:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:14:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:40 smithi151 ceph-mon[22663]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:40.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:40 smithi026 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:40.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:40 smithi026 ceph-mon[26568]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:41.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:41 smithi151 ceph-mon[22663]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:41.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:41 smithi026 ceph-mon[21867]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:41.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:41 smithi026 ceph-mon[26568]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:42.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:42 smithi151 ceph-mon[22663]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:42.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:42.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:42 smithi026 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:42.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:42.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:42 smithi026 ceph-mon[26568]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:42.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:44.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:43 smithi026 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:44.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:43 smithi026 ceph-mon[26568]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:44.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:43 smithi151 ceph-mon[22663]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:44.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:14:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:14:43] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-27T02:14:45.074 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-27T02:14:46.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:46 smithi026 ceph-mon[21867]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:46.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:46 smithi026 ceph-mon[26568]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:46.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:46 smithi151 ceph-mon[22663]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:47.604 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled to redeploy mgr.y on host 'smithi026' 2024-04-27T02:14:48.343 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps --refresh' 2024-04-27T02:14:48.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:48 smithi151 ceph-mon[22663]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:48.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:48 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2245582390' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:14:48.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:48 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:48 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:48 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:48 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:14:48.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[26568]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:48.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2245582390' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[21867]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2245582390' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:48.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:48 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:14:49.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:14:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:14:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:14:49.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:49 smithi026 ceph-mon[26568]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:14:49.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:49 smithi026 ceph-mon[26568]: Schedule redeploy daemon mgr.y 2024-04-27T02:14:49.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:49 smithi026 ceph-mon[21867]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:14:49.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:49 smithi026 ceph-mon[21867]: Schedule redeploy daemon mgr.y 2024-04-27T02:14:49.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:49 smithi151 ceph-mon[22663]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:14:49.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:49 smithi151 ceph-mon[22663]: Schedule redeploy daemon mgr.y 2024-04-27T02:14:50.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:50 smithi026 ceph-mon[21867]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:50.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:50 smithi026 ceph-mon[26568]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:50.535 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:14:50.535 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 66s ago 10m 17.2M - 0.23.0 ba2b418f427c 1a93078168a0 2024-04-27T02:14:50.535 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 error 66s ago 10m - - 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (119s) 66s ago 9m 50.4M - 3.5 e1d6a67b021e 0c8ecd76e92c 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283 running (4m) 66s ago 15m 510M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 210983b6ef19 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (16m) 66s ago 16m 410M - 17.2.0 e1d6a67b021e 56112248aa10 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (16m) 66s ago 17m 57.3M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (15m) 66s ago 15m 43.9M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (15m) 66s ago 15m 44.7M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 66s ago 11m 16.6M - 1.3.1 1dbe0e931976 b2e202622fa1 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (2m) 66s ago 11m 16.9M - 1.3.1 1dbe0e931976 00d8989305ef 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (14m) 66s ago 14m 51.8M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (14m) 66s ago 14m 52.9M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (14m) 66s ago 14m 49.2M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (13m) 66s ago 13m 54.3M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (13m) 66s ago 13m 52.7M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:14:50.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (13m) 66s ago 13m 52.2M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:14:50.537 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (12m) 66s ago 12m 48.2M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:14:50.537 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (12m) 66s ago 12m 51.7M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:14:50.537 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (2m) 66s ago 11m 64.8M - 2.33.4 514e6a882f6e fcc967e84146 2024-04-27T02:14:50.537 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (10m) 66s ago 10m 73.0M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:14:50.537 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (9m) 66s ago 9m 74.6M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:14:50.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:50 smithi151 ceph-mon[22663]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:14:51.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 180' 2024-04-27T02:14:51.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:51 smithi026 ceph-mon[21867]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:51.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:51 smithi026 ceph-mon[21867]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:14:51.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:51 smithi026 ceph-mon[26568]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:51.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:51 smithi026 ceph-mon[26568]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:14:51.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:51 smithi151 ceph-mon[22663]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:14:51.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:51 smithi151 ceph-mon[22663]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:14:52.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:52 smithi026 ceph-mon[26568]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:52.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:52 smithi026 ceph-mon[21867]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:52.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:52 smithi151 ceph-mon[22663]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:14:54.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:53 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:54.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:53 smithi151 ceph-mon[22663]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:54.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:14:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:14:53] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-27T02:14:54.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:53 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:54.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:53 smithi026 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:54.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:53 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:54.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:53 smithi026 ceph-mon[26568]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:56.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:56 smithi151 ceph-mon[22663]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:56.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:56 smithi026 ceph-mon[21867]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:56.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:56 smithi026 ceph-mon[26568]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:58.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:57 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:57 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:57 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:58.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:57 smithi151 ceph-mon[22663]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:58.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:57 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:57 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[21867]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[26568]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 67 op/s 2024-04-27T02:14:58.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:57 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:58.905 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:14:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:14:59.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[21867]: Deploying daemon mgr.y on smithi026 2024-04-27T02:14:59.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:14:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:14:58 smithi026 ceph-mon[26568]: Deploying daemon mgr.y on smithi026 2024-04-27T02:14:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:14:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:14:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-27T02:14:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-27T02:14:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:14:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:14:59.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:14:59.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:14:59.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:14:59.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:14:58 smithi151 ceph-mon[22663]: Deploying daemon mgr.y on smithi026 2024-04-27T02:15:00.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:00 smithi026 systemd[1]: Stopping Ceph mgr.y for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:15:00.677 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:00 smithi026 podman[60324]: 2024-04-27 02:15:00.438796844 +0000 UTC m=+0.148654603 container died 56112248aa1047ea368d2a2a3da8007327a3c936f9512f6c0b0099ffc68a4b4a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, architecture=x86_64, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, RELEASE=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-type=git, name=centos-stream, distribution-scope=public, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-27T02:15:01.086 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:00 smithi026 podman[60324]: 2024-04-27 02:15:00.798636239 +0000 UTC m=+0.508493999 container cleanup 56112248aa1047ea368d2a2a3da8007327a3c936f9512f6c0b0099ffc68a4b4a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, release=754, distribution-scope=public, vcs-type=git, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-27T02:15:01.086 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:00 smithi026 bash[60324]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y 2024-04-27T02:15:01.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:01 smithi026 ceph-mon[26568]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-27T02:15:01.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:01 smithi026 ceph-mon[26568]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:01.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:01 smithi026 ceph-mon[21867]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-27T02:15:01.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:01 smithi026 ceph-mon[21867]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:01.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 podman[60338]: 2024-04-27 02:15:01.108773124 +0000 UTC m=+0.669846591 container remove 56112248aa1047ea368d2a2a3da8007327a3c936f9512f6c0b0099ffc68a4b4a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, architecture=x86_64, version=8, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, distribution-scope=public) 2024-04-27T02:15:01.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:15:01.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-27T02:15:01.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 systemd[1]: Stopped Ceph mgr.y for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:15:01.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 3.666s CPU time. 2024-04-27T02:15:01.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:01 smithi151 ceph-mon[22663]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-27T02:15:01.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:01 smithi151 ceph-mon[22663]: from='client.15174 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:01.780 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 systemd[1]: Starting Ceph mgr.y for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:15:01.781 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 podman[60470]: 2024-04-27 02:15:01.676248392 +0000 UTC m=+0.096168896 container create b4373d6f87916ff93762fcf262c4106e6c03f2082c1a66ce90321c11a0c70e5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb) 2024-04-27T02:15:01.781 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 podman[60470]: 2024-04-27 02:15:01.59871435 +0000 UTC m=+0.018634857 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:15:02.259 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 podman[60470]: 2024-04-27 02:15:01.920846619 +0000 UTC m=+0.340767124 container init b4373d6f87916ff93762fcf262c4106e6c03f2082c1a66ce90321c11a0c70e5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_CLEAN=True) 2024-04-27T02:15:02.260 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 podman[60470]: 2024-04-27 02:15:01.923602575 +0000 UTC m=+0.343523082 container start b4373d6f87916ff93762fcf262c4106e6c03f2082c1a66ce90321c11a0c70e5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:15:02.260 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:01 smithi026 ceph-mgr[60490]: -- 172.21.15.26:0/2669485616 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fcc8bec5a0 con 0x55fcc8bc0800 2024-04-27T02:15:02.260 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:02 smithi026 bash[60470]: b4373d6f87916ff93762fcf262c4106e6c03f2082c1a66ce90321c11a0c70e5e 2024-04-27T02:15:02.260 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:02 smithi026 systemd[1]: Started Ceph mgr.y for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:15:02.260 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:02.083+0000 7fb066c7d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:15:02.260 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:02.281+0000 7fb066c7d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:15:02.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:02.392+0000 7fb066c7d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:15:02.943 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:02.734+0000 7fb066c7d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:15:02.943 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:02.823+0000 7fb066c7d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:03.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:03.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:03.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:03 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:15:04.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:15:03] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-27T02:15:04.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:04.122+0000 7fb066c7d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:15:04.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:04.307+0000 7fb066c7d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:15:04.712 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:04.491+0000 7fb066c7d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:15:04.997 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:04.800+0000 7fb066c7d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:15:04.997 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:04.915+0000 7fb066c7d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:15:05.304 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-mon[22663]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-04-27T02:15:05.304 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:15:05.324+0000 7f80bd33e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:15:05.305 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:05.306 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:05.306 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:05.306 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:05.306 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:05.306 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:05.306 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return future.result(timeout) 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:15:05.307 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.__get_result() 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: raise self._exception 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: config_blobs=daemon_spec.final_config, 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:15:05.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:05.309 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:15:05.309 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:05.309 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:15:05.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:05 smithi026 ceph-mon[21867]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-04-27T02:15:05.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:05 smithi026 ceph-mon[26568]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-04-27T02:15:05.320 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:05 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:05.128+0000 7fb066c7d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:15:05.320 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:05 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:05.342+0000 7fb066c7d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:05.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:05 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:06 smithi151 ceph-mon[22663]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:15:06.322 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:15:06.323 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: return future.result(timeout) 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.__get_result() 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: raise self._exception 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:06.324 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:06.325 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:06.326 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:06.326 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:06.326 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:06.326 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.326 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.326 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:06 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:15:06.357 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:06.004+0000 7fb066c7d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:15:06.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-mon[26568]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:06.357 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:06.358 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:15:06.359 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:06.360 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:06.361 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:15:06.362 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:15:06.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:15:06.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:15:06.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:15:06.366 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:15:06.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:15:06.776 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:06.464+0000 7fb066c7d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:15:06.776 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:06.579+0000 7fb066c7d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:15:06.776 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:06.798+0000 7fb066c7d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:15:07.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:07 smithi026 ceph-mon[26568]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-04-27T02:15:07.344 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:07 smithi026 ceph-mon[21867]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-04-27T02:15:07.345 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:07.366+0000 7fb066c7d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:15:07.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:07 smithi151 ceph-mon[22663]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-04-27T02:15:07.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:07.472+0000 7fb066c7d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:15:07.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:07.758+0000 7fb066c7d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:15:07.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:07.904+0000 7fb066c7d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:15:07.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:08.007+0000 7fb066c7d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:15:08.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:08.131+0000 7fb066c7d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:15:08.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:08.252+0000 7fb066c7d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:15:08.906 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:15:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:15:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:15:09.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:08 smithi151 ceph-mon[22663]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-04-27T02:15:09.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:08 smithi026 ceph-mon[21867]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-04-27T02:15:09.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:08 smithi026 ceph-mon[26568]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-04-27T02:15:09.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:09.145+0000 7fb066c7d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:15:09.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:09.386+0000 7fb066c7d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:15:09.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:15:09] ENGINE Bus STARTING 2024-04-27T02:15:09.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: CherryPy Checker: 2024-04-27T02:15:09.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: The Application mounted at '' has an empty config. 2024-04-27T02:15:09.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:15:09.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:15:09] ENGINE Serving on http://:::9283 2024-04-27T02:15:09.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:15:09 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:15:09] ENGINE Bus STARTED 2024-04-27T02:15:10.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 645 B/s rd, 0 op/s 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 787 B/s rd, 0 op/s 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:15:10.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: Standby manager daemon y restarted 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: Standby manager daemon y started 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:15:10.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 645 B/s rd, 0 op/s 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 787 B/s rd, 0 op/s 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: Standby manager daemon y restarted 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: Standby manager daemon y started 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:15:10.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:15:10.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:10 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:15:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:15:10.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 645 B/s rd, 0 op/s 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 787 B/s rd, 0 op/s 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: Standby manager daemon y restarted 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: Standby manager daemon y started 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:15:10.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:10 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:15:11.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:11 smithi026 ceph-mon[21867]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:15:11.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:11 smithi026 ceph-mon[21867]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-27T02:15:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:11 smithi026 ceph-mon[26568]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:15:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:11 smithi026 ceph-mon[26568]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-27T02:15:11.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:11 smithi151 ceph-mon[22663]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:15:11.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:11 smithi151 ceph-mon[22663]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-27T02:15:12.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:12 smithi026 ceph-mon[21867]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 788 B/s rd, 0 op/s 2024-04-27T02:15:12.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:12 smithi026 ceph-mon[26568]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 788 B/s rd, 0 op/s 2024-04-27T02:15:12.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:12 smithi151 ceph-mon[22663]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 788 B/s rd, 0 op/s 2024-04-27T02:15:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:15:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:15:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:15:13.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:15:13.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:15:13.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4177598741' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:15:13.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4177598741' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:13.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:15:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:15:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:15:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:15:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:15:13.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:13 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4177598741' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:15:14.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:14 smithi151 ceph-mon[22663]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 337 B/s rd, 0 op/s 2024-04-27T02:15:14.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:15:13] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-27T02:15:14.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:14 smithi026 ceph-mon[21867]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 337 B/s rd, 0 op/s 2024-04-27T02:15:14.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:14 smithi026 ceph-mon[26568]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 337 B/s rd, 0 op/s 2024-04-27T02:15:16.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:16 smithi151 ceph-mon[22663]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 337 B/s rd, 0 op/s 2024-04-27T02:15:16.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:16 smithi026 ceph-mon[21867]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 337 B/s rd, 0 op/s 2024-04-27T02:15:16.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:16 smithi026 ceph-mon[26568]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 337 B/s rd, 0 op/s 2024-04-27T02:15:18.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:17 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:17 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:17 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:18.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:17 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:15:18.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:17 smithi151 ceph-mon[22663]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 117 B/s rd, 0 op/s 2024-04-27T02:15:18.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:17 smithi151 ceph-mon[22663]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 134 B/s rd, 0 op/s 2024-04-27T02:15:18.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:17 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[26568]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 117 B/s rd, 0 op/s 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[26568]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 134 B/s rd, 0 op/s 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[21867]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 117 B/s rd, 0 op/s 2024-04-27T02:15:18.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[21867]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 134 B/s rd, 0 op/s 2024-04-27T02:15:18.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:17 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:15:19.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:18 smithi151 ceph-mon[22663]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:15:19.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:15:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:15:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:15:19.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:18 smithi026 ceph-mon[21867]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:15:19.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:18 smithi026 ceph-mon[26568]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:15:20.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:19 smithi151 ceph-mon[22663]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-27T02:15:20.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:19 smithi026 ceph-mon[21867]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-27T02:15:20.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:19 smithi026 ceph-mon[26568]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-27T02:15:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:21 smithi151 ceph-mon[22663]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-27T02:15:22.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:21 smithi026 ceph-mon[21867]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-27T02:15:22.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:21 smithi026 ceph-mon[26568]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-27T02:15:22.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:22 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:22.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:22 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:23.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:22 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:24.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:23 smithi026 ceph-mon[21867]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-27T02:15:24.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:23 smithi026 ceph-mon[26568]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-27T02:15:24.244 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:23 smithi151 ceph-mon[22663]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-27T02:15:24.245 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:15:23] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-27T02:15:26.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:25 smithi151 ceph-mon[22663]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:15:26.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:25 smithi026 ceph-mon[21867]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:15:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:25 smithi026 ceph-mon[26568]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:15:28.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:27 smithi151 ceph-mon[22663]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:15:28.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:27 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:28.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:27 smithi026 ceph-mon[21867]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:15:28.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:27 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:28.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:27 smithi026 ceph-mon[26568]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:15:28.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:27 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:29.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:15:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:15:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:15:30.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:29 smithi151 ceph-mon[22663]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:29 smithi026 ceph-mon[21867]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:29 smithi026 ceph-mon[26568]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:32.250 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:31 smithi026 ceph-mon[21867]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:32.250 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:31 smithi026 ceph-mon[26568]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:31 smithi151 ceph-mon[22663]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:33.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:32 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:33.293 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:32 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:33.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:32 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:34.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:33 smithi151 ceph-mon[22663]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:34.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:15:33] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-27T02:15:34.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:33 smithi026 ceph-mon[21867]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:34.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:33 smithi026 ceph-mon[26568]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:36.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:35 smithi151 ceph-mon[22663]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:36.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:35 smithi026 ceph-mon[21867]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:36.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:35 smithi026 ceph-mon[26568]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:38.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:37 smithi151 ceph-mon[22663]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:38.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:37 smithi026 ceph-mon[21867]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:38.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:37 smithi026 ceph-mon[26568]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:39.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:15:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:15:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:15:40.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:39 smithi151 ceph-mon[22663]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:40.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:39 smithi026 ceph-mon[21867]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:40.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:39 smithi026 ceph-mon[26568]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:42.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:41 smithi026 ceph-mon[21867]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:42.261 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:41 smithi026 ceph-mon[26568]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:42.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:41 smithi151 ceph-mon[22663]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:42 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:43.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:43.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:42 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:43.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:43.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:42 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:44.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:43 smithi151 ceph-mon[22663]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:44.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:15:43] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-27T02:15:44.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:43 smithi026 ceph-mon[21867]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:44.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:43 smithi026 ceph-mon[26568]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:46.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:45 smithi151 ceph-mon[22663]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:46.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:45 smithi026 ceph-mon[21867]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:46.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:45 smithi026 ceph-mon[26568]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:48.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:47 smithi151 ceph-mon[22663]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:47 smithi026 ceph-mon[21867]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:48.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:47 smithi026 ceph-mon[26568]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:49.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:15:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:15:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:15:50.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:49 smithi151 ceph-mon[22663]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:50.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:49 smithi026 ceph-mon[21867]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:50.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:49 smithi026 ceph-mon[26568]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:52.272 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:51 smithi026 ceph-mon[26568]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:52.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:51 smithi026 ceph-mon[21867]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:52.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:51 smithi151 ceph-mon[22663]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:53.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:52 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:53.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:52 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:53.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:52 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:15:54.248 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:53 smithi151 ceph-mon[22663]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:54.248 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:15:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:15:53] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-27T02:15:54.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:53 smithi026 ceph-mon[21867]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:54.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:53 smithi026 ceph-mon[26568]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:56.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:55 smithi151 ceph-mon[22663]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:56.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:55 smithi026 ceph-mon[21867]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:56.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:55 smithi026 ceph-mon[26568]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:15:58.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:57 smithi151 ceph-mon[22663]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:58.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:57 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:58.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:57 smithi026 ceph-mon[21867]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:58.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:57 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:58.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:57 smithi026 ceph-mon[26568]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:15:58.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:57 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:15:59.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:15:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:15:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:16:00.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:15:59 smithi151 ceph-mon[22663]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:00.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:15:59 smithi026 ceph-mon[21867]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:15:59 smithi026 ceph-mon[26568]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:02.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:01 smithi026 ceph-mon[21867]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:02.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:01 smithi026 ceph-mon[26568]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:02.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:01 smithi151 ceph-mon[22663]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:03.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:02 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:03.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:02 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:03.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:02 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:04.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:03 smithi151 ceph-mon[22663]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:04.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:16:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:16:03] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-27T02:16:04.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:03 smithi026 ceph-mon[21867]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:04.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:03 smithi026 ceph-mon[26568]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:06.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:06 smithi151 ceph-mon[22663]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:06.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:06 smithi026 ceph-mon[21867]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:06.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:06 smithi026 ceph-mon[26568]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:08.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:08 smithi151 ceph-mon[22663]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:08.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:08 smithi026 ceph-mon[21867]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:08.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:08 smithi026 ceph-mon[26568]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:09.096 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:16:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:16:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:16:10.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:10 smithi151 ceph-mon[22663]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:10.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:10 smithi026 ceph-mon[21867]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:10.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:10 smithi026 ceph-mon[26568]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:12.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:12 smithi026 ceph-mon[21867]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:12.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:12 smithi026 ceph-mon[26568]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:12.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:12 smithi151 ceph-mon[22663]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:13.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:13 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:13.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:13 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:13.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:13 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:13.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:13 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:13 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:13.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:13 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:14.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:14 smithi151 ceph-mon[22663]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:14.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:16:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:16:13] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-27T02:16:14.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:14 smithi026 ceph-mon[21867]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:14.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:14 smithi026 ceph-mon[26568]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:16.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:16 smithi151 ceph-mon[22663]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:16.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:16 smithi026 ceph-mon[21867]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:16 smithi026 ceph-mon[26568]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:17.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:17 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:16:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:17 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:16:17.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:17 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:16:18.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:18 smithi151 ceph-mon[22663]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:18.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:18 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:16:18.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:18 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:16:18.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:18 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:16:18.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[21867]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:18.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:16:18.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:16:18.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:16:18.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[26568]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:18.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:16:18.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:16:18.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:18 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:16:19.007 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:16:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:16:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:16:19.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:19 smithi151 ceph-mon[22663]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-27T02:16:19.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:19 smithi151 ceph-mon[22663]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-27T02:16:19.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:19 smithi026 ceph-mon[21867]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-27T02:16:19.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:19 smithi026 ceph-mon[21867]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-27T02:16:19.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:19 smithi026 ceph-mon[26568]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-27T02:16:19.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:19 smithi026 ceph-mon[26568]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-27T02:16:21.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:21 smithi151 ceph-mon[22663]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:21.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:21 smithi026 ceph-mon[21867]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:21.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:21 smithi026 ceph-mon[26568]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:23.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:23 smithi151 ceph-mon[22663]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:23.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:23 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:23.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:23 smithi026 ceph-mon[21867]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:23.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:23 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:23.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:23 smithi026 ceph-mon[26568]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:23.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:23 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:24.248 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:16:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:16:23] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-27T02:16:25.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:25 smithi151 ceph-mon[22663]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:25.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:25 smithi026 ceph-mon[21867]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:25.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:25 smithi026 ceph-mon[26568]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:27.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:27 smithi151 ceph-mon[22663]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:27.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:27 smithi026 ceph-mon[26568]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:27.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:27 smithi026 ceph-mon[21867]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:16:28.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:28 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:28.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:28 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:28.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:28 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:29.024 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:16:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:16:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:16:29.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:29 smithi151 ceph-mon[22663]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:16:29.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:29 smithi026 ceph-mon[21867]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:16:29.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:29 smithi026 ceph-mon[26568]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:16:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:31 smithi151 ceph-mon[22663]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:31.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:31 smithi026 ceph-mon[21867]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:31 smithi026 ceph-mon[26568]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:33.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:33 smithi151 ceph-mon[22663]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:33.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:33 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:33.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:33 smithi026 ceph-mon[21867]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:33.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:33 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:33.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:33 smithi026 ceph-mon[26568]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:33.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:33 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:34.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:16:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:16:33] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-04-27T02:16:35.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:35 smithi151 ceph-mon[22663]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:35.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:35 smithi026 ceph-mon[21867]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:35.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:35 smithi026 ceph-mon[26568]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:37.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:37 smithi151 ceph-mon[22663]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:37.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:37 smithi026 ceph-mon[26568]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:37.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:37 smithi026 ceph-mon[21867]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:39.037 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:16:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:16:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:16:39.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:39 smithi151 ceph-mon[22663]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:39.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:39 smithi026 ceph-mon[21867]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:39.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:39 smithi026 ceph-mon[26568]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:41.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:41 smithi151 ceph-mon[22663]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:41.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:41 smithi026 ceph-mon[21867]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:41.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:41 smithi026 ceph-mon[26568]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:43.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:43 smithi151 ceph-mon[22663]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:43 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:43 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:43.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:43 smithi026 ceph-mon[26568]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:43.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:43 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:43.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:43 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:43.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:43 smithi026 ceph-mon[21867]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:43.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:43 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:43.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:43 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:44.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:16:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:16:43] "GET /metrics HTTP/1.1" 200 37971 "" "Prometheus/2.33.4" 2024-04-27T02:16:45.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:45 smithi151 ceph-mon[22663]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:45.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:45 smithi026 ceph-mon[21867]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:45.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:45 smithi026 ceph-mon[26568]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:47.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:47 smithi151 ceph-mon[22663]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:47.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:47 smithi026 ceph-mon[21867]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:47.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:47 smithi026 ceph-mon[26568]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:49.050 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:16:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:16:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:16:49.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:49 smithi151 ceph-mon[22663]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:49.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:49 smithi026 ceph-mon[21867]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:49.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:49 smithi026 ceph-mon[26568]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:51.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:51 smithi151 ceph-mon[22663]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:51.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:51 smithi026 ceph-mon[21867]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:51.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:51 smithi026 ceph-mon[26568]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:53.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:53 smithi151 ceph-mon[22663]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:53.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:53 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:53.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:53 smithi026 ceph-mon[21867]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:53.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:53 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:53.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:53 smithi026 ceph-mon[26568]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:53.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:53 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:16:54.248 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:16:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:16:53] "GET /metrics HTTP/1.1" 200 37971 "" "Prometheus/2.33.4" 2024-04-27T02:16:55.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:55 smithi151 ceph-mon[22663]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:55.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:55 smithi026 ceph-mon[26568]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:55.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:55 smithi026 ceph-mon[21867]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:16:57.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:57 smithi151 ceph-mon[22663]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:57.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:57 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:57.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:57 smithi026 ceph-mon[26568]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:57.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:57 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:57.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:57 smithi026 ceph-mon[21867]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:57.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:57 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:16:59.064 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:16:58 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:16:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:16:59.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:16:59 smithi151 ceph-mon[22663]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:59.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:16:59 smithi026 ceph-mon[21867]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:16:59.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:16:59 smithi026 ceph-mon[26568]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:01 smithi151 ceph-mon[22663]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:01.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:01 smithi026 ceph-mon[21867]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:01.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:01 smithi026 ceph-mon[26568]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:03.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:03 smithi151 ceph-mon[22663]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:03.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:03 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:03.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:03 smithi026 ceph-mon[21867]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:03.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:03 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:03.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:03 smithi026 ceph-mon[26568]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:03.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:03 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:04.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:17:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:17:03] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-27T02:17:05.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:05 smithi151 ceph-mon[22663]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:05.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:05 smithi026 ceph-mon[21867]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:05.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:05 smithi026 ceph-mon[26568]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:07.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:07 smithi151 ceph-mon[22663]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:07.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:07 smithi026 ceph-mon[21867]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:07.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:07 smithi026 ceph-mon[26568]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:09.078 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:17:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:17:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-27T02:17:09.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:09 smithi151 ceph-mon[22663]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:09.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:09 smithi026 ceph-mon[21867]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:09.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:09 smithi026 ceph-mon[26568]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:11.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:11 smithi151 ceph-mon[22663]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:11.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:11 smithi026 ceph-mon[21867]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:11.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:11 smithi026 ceph-mon[26568]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:12.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:12 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:12.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:12 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:12.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:12 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:13.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:13 smithi151 ceph-mon[22663]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:13.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:13 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:13.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:13 smithi026 ceph-mon[26568]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:13.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:13 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:13.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:13 smithi026 ceph-mon[21867]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:13.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:13 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:14.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:17:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:17:13] "GET /metrics HTTP/1.1" 200 37971 "" "Prometheus/2.33.4" 2024-04-27T02:17:15.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:15 smithi151 ceph-mon[22663]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:15.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:15 smithi026 ceph-mon[21867]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:15.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:15 smithi026 ceph-mon[26568]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:17.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:17 smithi151 ceph-mon[22663]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:17 smithi026 ceph-mon[21867]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:17.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:17 smithi026 ceph-mon[26568]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:18.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:18 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:17:18.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:18 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:17:18.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:18 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:17:19.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[21867]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[21867]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[21867]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[21867]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[26568]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[26568]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[26568]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:19.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:19 smithi026 ceph-mon[26568]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:17:19.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:19 smithi151 ceph-mon[22663]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:19.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:19 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:17:19.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:19 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:17:19.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:19 smithi151 ceph-mon[22663]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:17:19.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:19 smithi151 ceph-mon[22663]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:19.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:19 smithi151 ceph-mon[22663]: from='mgr.24785 ' entity='mgr.x' 2024-04-27T02:17:21.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:21 smithi151 ceph-mon[22663]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:21.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:21 smithi026 ceph-mon[26568]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:21.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:21 smithi026 ceph-mon[21867]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:23.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:23 smithi151 ceph-mon[22663]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-27T02:17:23.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:23 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:23.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:23 smithi026 ceph-mon[21867]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-27T02:17:23.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:23 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:23.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:23 smithi026 ceph-mon[26568]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-27T02:17:23.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:23 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:24.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:17:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:17:23] "GET /metrics HTTP/1.1" 200 37971 "" "Prometheus/2.33.4" 2024-04-27T02:17:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:25 smithi151 ceph-mon[22663]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:25.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:25 smithi026 ceph-mon[21867]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:25.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:25 smithi026 ceph-mon[26568]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:27.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:27 smithi151 ceph-mon[22663]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:27.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:27 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:27.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:27 smithi026 ceph-mon[21867]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:27.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:27 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:27.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:27 smithi026 ceph-mon[26568]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:27.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:27 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:29.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:28 smithi151 ceph-mon[22663]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:17:29.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:28 smithi026 ceph-mon[21867]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:17:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:28 smithi026 ceph-mon[26568]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:17:31.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:31 smithi151 ceph-mon[22663]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:31.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:31 smithi026 ceph-mon[21867]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:31.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:31 smithi026 ceph-mon[26568]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:33.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:33 smithi151 ceph-mon[22663]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:33.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:33 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:33.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:33 smithi026 ceph-mon[21867]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:33.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:33 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:33.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:33 smithi026 ceph-mon[26568]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:33.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:33 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:34.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:17:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:17:33] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-27T02:17:35.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:35 smithi151 ceph-mon[22663]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:35.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:35 smithi026 ceph-mon[21867]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:35.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:35 smithi026 ceph-mon[26568]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:37.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:37 smithi151 ceph-mon[22663]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:37.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:37 smithi026 ceph-mon[21867]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:37.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:37 smithi026 ceph-mon[26568]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:39.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:38 smithi151 ceph-mon[22663]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:39.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:38 smithi026 ceph-mon[21867]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:39.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:38 smithi026 ceph-mon[26568]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:41.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:41 smithi151 ceph-mon[22663]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:41.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:41 smithi026 ceph-mon[21867]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:41 smithi026 ceph-mon[26568]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:42.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:42 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:42.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:42 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:42.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:42 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:43.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:43 smithi151 ceph-mon[22663]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:43.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:43 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:43.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:43 smithi026 ceph-mon[21867]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:43.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:43 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:43.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:43 smithi026 ceph-mon[26568]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:43.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:43 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:44.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:17:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:17:43] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-27T02:17:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:45 smithi151 ceph-mon[22663]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:45.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:45 smithi026 ceph-mon[21867]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:45 smithi026 ceph-mon[26568]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:47.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:47 smithi151 ceph-mon[22663]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:47.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:47 smithi026 ceph-mon[21867]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:47.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:47 smithi026 ceph-mon[26568]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:49.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:48 smithi151 ceph-mon[22663]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:49.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:48 smithi026 ceph-mon[21867]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:49.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:48 smithi026 ceph-mon[26568]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:51.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:51 smithi151 ceph-mon[22663]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:51.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:51 smithi026 ceph-mon[21867]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:51.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:51 smithi026 ceph-mon[26568]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:53.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:53 smithi151 ceph-mon[22663]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:53.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:53 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:53.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:53 smithi026 ceph-mon[21867]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:53.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:53 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:53.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:53 smithi026 ceph-mon[26568]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:53.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:53 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:17:54.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:17:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:17:53] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-27T02:17:54.722 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:17:55.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:55 smithi151 ceph-mon[22663]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:55.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:55 smithi026 ceph-mon[21867]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:55.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:55 smithi026 ceph-mon[26568]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:56.843 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:17:56.843 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 2m ago 13m 24.8M - 0.23.0 ba2b418f427c 1a93078168a0 2024-04-27T02:17:56.843 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 error 2m ago 13m - - 2024-04-27T02:17:56.843 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (2m) 2m ago 12m 45.9M - 3.5 e1d6a67b021e 09155a3d60fb 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283 running (8m) 2m ago 18m 512M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 210983b6ef19 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (2m) 2m ago 20m 450M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (20m) 2m ago 20m 62.3M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (18m) 2m ago 18m 46.1M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (18m) 2m ago 18m 48.0M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 2m ago 14m 18.0M - 1.3.1 1dbe0e931976 b2e202622fa1 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (5m) 2m ago 14m 20.1M - 1.3.1 1dbe0e931976 00d8989305ef 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (18m) 2m ago 18m 52.7M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (17m) 2m ago 17m 54.1M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (17m) 2m ago 17m 49.8M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (16m) 2m ago 16m 55.6M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (16m) 2m ago 16m 53.6M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:17:56.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (16m) 2m ago 16m 53.2M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:17:56.845 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (15m) 2m ago 15m 49.3M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:17:56.845 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (15m) 2m ago 15m 53.2M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:17:56.845 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (5m) 2m ago 14m 69.1M - 2.33.4 514e6a882f6e fcc967e84146 2024-04-27T02:17:56.845 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (13m) 2m ago 13m 73.4M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:17:56.845 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (13m) 2m ago 13m 74.9M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:17:57.435 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-27T02:17:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:57 smithi151 ceph-mon[22663]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:57 smithi151 ceph-mon[22663]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:57.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:57 smithi026 ceph-mon[21867]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:57.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:57 smithi026 ceph-mon[21867]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:57 smithi026 ceph-mon[26568]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:17:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:57 smithi026 ceph-mon[26568]: from='mgr.24785 172.21.15.151:0/2992886609' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:17:58.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:58 smithi026 ceph-mon[26568]: from='client.25138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:17:58.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:58 smithi026 ceph-mon[21867]: from='client.25138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:17:58.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:58 smithi151 ceph-mon[22663]: from='client.25138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:17:59.340 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:17:59.341 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:17:59.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:17:59 smithi151 ceph-mon[22663]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:59.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:17:59 smithi026 ceph-mon[21867]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:17:59.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:17:59 smithi026 ceph-mon[26568]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:00.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:18:00.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:00 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/993423385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:18:00.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:00 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/993423385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:18:00.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:00 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/993423385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:18:01.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:01 smithi026 ceph-mon[21867]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:01.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:01 smithi026 ceph-mon[26568]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:01.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:01 smithi151 ceph-mon[22663]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:02.083 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-27T02:18:02.083 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-27T02:18:02.083 INFO:teuthology.orchestra.run.smithi026.stdout: daemon grafana.a on smithi151 is in error state 2024-04-27T02:18:02.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:02 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1700828385' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:18:02.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:02 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1700828385' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:18:02.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:02 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1700828385' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:18:02.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph -s' 2024-04-27T02:18:03.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:03 smithi151 ceph-mon[22663]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:03.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:03 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:03.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:03 smithi026 ceph-mon[21867]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:03.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:03 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:03.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:03 smithi026 ceph-mon[26568]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:03.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:03 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:04.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:18:03] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-27T02:18:04.779 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-27T02:18:04.779 INFO:teuthology.orchestra.run.smithi026.stdout: id: 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:18:04.779 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_WARN 2024-04-27T02:18:04.779 INFO:teuthology.orchestra.run.smithi026.stdout: 1 failed cephadm daemon(s) 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: x(active, since 6m), standbys: y 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-27T02:18:04.780 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 241 objects, 585 KiB 2024-04-27T02:18:04.781 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:04.781 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-27T02:18:04.781 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:18:04.781 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-27T02:18:04.781 INFO:teuthology.orchestra.run.smithi026.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-27T02:18:04.781 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:18:05.431 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph mgr fail' 2024-04-27T02:18:05.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:05 smithi151 ceph-mon[22663]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:05.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:05 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4072569737' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:18:05.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:05 smithi026 ceph-mon[26568]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:05.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:05 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4072569737' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:18:05.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:05 smithi026 ceph-mon[21867]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:05.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:05 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4072569737' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:18:07.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-mon[22663]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:07.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3850491757' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:18:07.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:18:07.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-mon[22663]: osdmap e102: 8 total, 8 up, 8 in 2024-04-27T02:18:07.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:07.364+0000 7f812ad37700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-27T02:18:07.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ignoring --setuser ceph since I am not root 2024-04-27T02:18:07.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ignoring --setgroup ceph since I am not root 2024-04-27T02:18:07.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-mgr[43038]: -- 172.21.15.151:0/2093412087 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561c48aa25a0 con 0x561c48a77000 2024-04-27T02:18:07.599 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:07.555+0000 7f73495ee200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:18:07.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[21867]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:07.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3850491757' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:18:07.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:18:07.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[21867]: osdmap e102: 8 total, 8 up, 8 in 2024-04-27T02:18:07.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[26568]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3850491757' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-mon[26568]: osdmap e102: 8 total, 8 up, 8 in 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:07] ENGINE Bus STOPPING 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:07] ENGINE Bus STOPPED 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:07] ENGINE Bus STARTING 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:07] ENGINE Serving on http://:::9283 2024-04-27T02:18:07.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:07 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:07] ENGINE Bus STARTED 2024-04-27T02:18:08.035 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 180' 2024-04-27T02:18:08.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:07.742+0000 7f73495ee200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:18:08.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:07.852+0000 7f73495ee200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:18:08.544 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:08.182+0000 7f73495ee200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:18:08.544 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:08.278+0000 7f73495ee200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: mgrmap e28: y(active, starting, since 0.0190856s) 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:18:08.545 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: Manager daemon y is now available 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: [27/Apr/2024:02:18:07] ENGINE Bus STARTING 2024-04-27T02:18:08.546 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:08 smithi151 ceph-mon[22663]: [27/Apr/2024:02:18:07] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: mgrmap e28: y(active, starting, since 0.0190856s) 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:18:08.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: Manager daemon y is now available 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:18:08.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: [27/Apr/2024:02:18:07] ENGINE Bus STARTING 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[26568]: [27/Apr/2024:02:18:07] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: mgrmap e28: y(active, starting, since 0.0190856s) 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:18:08.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: Manager daemon y is now available 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:18:08.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: [27/Apr/2024:02:18:07] ENGINE Bus STARTING 2024-04-27T02:18:08.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:08 smithi026 ceph-mon[21867]: [27/Apr/2024:02:18:07] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:18:09.505 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:09 smithi151 ceph-mon[22663]: [27/Apr/2024:02:18:08] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:18:09.506 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:09 smithi151 ceph-mon[22663]: [27/Apr/2024:02:18:08] ENGINE Bus STARTED 2024-04-27T02:18:09.506 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:09 smithi151 ceph-mon[22663]: mgrmap e29: y(active, since 1.03391s) 2024-04-27T02:18:09.506 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:09 smithi151 ceph-mon[22663]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:09.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[21867]: [27/Apr/2024:02:18:08] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:18:09.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[21867]: [27/Apr/2024:02:18:08] ENGINE Bus STARTED 2024-04-27T02:18:09.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[21867]: mgrmap e29: y(active, since 1.03391s) 2024-04-27T02:18:09.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:09.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[26568]: [27/Apr/2024:02:18:08] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:18:09.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[26568]: [27/Apr/2024:02:18:08] ENGINE Bus STARTED 2024-04-27T02:18:09.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[26568]: mgrmap e29: y(active, since 1.03391s) 2024-04-27T02:18:09.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:09 smithi026 ceph-mon[26568]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:09.846 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:09 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:09.528+0000 7f73495ee200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:18:09.846 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:09 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:09.702+0000 7f73495ee200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:18:10.184 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:09 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:09.892+0000 7f73495ee200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:18:10.184 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:10.206+0000 7f73495ee200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:18:10.501 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:10 smithi151 ceph-mon[22663]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:10.501 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:10 smithi151 ceph-mon[22663]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:18:10.502 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:10 smithi151 ceph-mon[22663]: Cluster is now healthy 2024-04-27T02:18:10.502 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:10.309+0000 7f73495ee200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:18:10.502 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:10.524+0000 7f73495ee200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:18:10.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:10 smithi026 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:10.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:10 smithi026 ceph-mon[21867]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:18:10.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:10 smithi026 ceph-mon[21867]: Cluster is now healthy 2024-04-27T02:18:10.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:10 smithi026 ceph-mon[26568]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:10.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:10 smithi026 ceph-mon[26568]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:18:10.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:10 smithi026 ceph-mon[26568]: Cluster is now healthy 2024-04-27T02:18:10.846 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:10.739+0000 7f73495ee200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:18:11.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:11 smithi151 ceph-mon[22663]: mgrmap e30: y(active, since 3s) 2024-04-27T02:18:11.596 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:11 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:11.359+0000 7f73495ee200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:18:11.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:11 smithi026 ceph-mon[21867]: mgrmap e30: y(active, since 3s) 2024-04-27T02:18:11.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:11 smithi026 ceph-mon[26568]: mgrmap e30: y(active, since 3s) 2024-04-27T02:18:12.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:11 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:11.829+0000 7f73495ee200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:18:12.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:11 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:11.933+0000 7f73495ee200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:18:12.596 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:12.147+0000 7f73495ee200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:18:13.027 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:12.704+0000 7f73495ee200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:18:13.027 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:12.803+0000 7f73495ee200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:18:13.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:12 smithi151 ceph-mon[22663]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:13.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:12 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:13.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:12 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:12 smithi151 ceph-mon[22663]: mgrmap e31: y(active, since 5s) 2024-04-27T02:18:13.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[21867]: mgrmap e31: y(active, since 5s) 2024-04-27T02:18:13.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[26568]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:13.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:13.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:13.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:12 smithi026 ceph-mon[26568]: mgrmap e31: y(active, since 5s) 2024-04-27T02:18:13.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:13.049+0000 7f73495ee200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:18:13.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:13.186+0000 7f73495ee200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:18:13.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:13.282+0000 7f73495ee200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:18:13.758 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:13.399+0000 7f73495ee200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:18:13.758 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:13.494+0000 7f73495ee200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:18:14.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-mon[22663]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:14.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:13 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[26568]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[21867]: from='client.24983 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:13 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:14.596 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:14.313+0000 7f73495ee200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:18:14.596 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:14.534+0000 7f73495ee200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:18:14.596 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:18:14] ENGINE Bus STARTING 2024-04-27T02:18:14.605 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=warn ts=2024-04-27T02:18:14.255Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:18:14.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CherryPy Checker: 2024-04-27T02:18:14.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: The Application mounted at '' has an empty config. 2024-04-27T02:18:14.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:18:14.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:18:14] ENGINE Serving on http://:::9283 2024-04-27T02:18:14.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:18:14] ENGINE Bus STARTED 2024-04-27T02:18:15.105 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:18:14.830Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=2 2024-04-27T02:18:15.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:15.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T02:18:15.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: Standby manager daemon x started 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:18:15.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:18:15.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:18:15.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:18:15.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:15.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:15.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:15.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:14 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:18:15.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:18:15.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: Standby manager daemon x started 2024-04-27T02:18:15.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:18:15.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:18:15.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:18:15.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:18:15.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:15.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:15.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:15.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:14 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:18:16.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:18:16.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:18:16.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: mgrmap e32: y(active, since 7s), standbys: x 2024-04-27T02:18:16.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:18:16.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:15 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: mgrmap e32: y(active, since 7s), standbys: x 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: mgrmap e32: y(active, since 7s), standbys: x 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:16.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:15 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:17.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:18:17.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:18:17.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:18:17.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:18:17.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:18:17.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:18:17.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:18:17.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-27T02:18:17.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:18:17.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:18:17.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:18:17.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:16 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:18:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:18:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:18:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:18:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:18:17.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:18:17.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:18:17.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:18:17.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-27T02:18:17.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:18:17.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:18:17.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:18:17.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:16 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:18.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:17 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:18:18.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:17 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:18:18.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:17 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:18:18.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:17 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:18:18.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:17 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:18:18.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:17 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:18:18.963 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:18.720Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.151:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.151:8765: connect: connection refused" 2024-04-27T02:18:18.963 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:18.721Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.151:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.151:8765: connect: connection refused" 2024-04-27T02:18:18.963 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:18.722Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.151:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.151:8765: connect: connection refused" 2024-04-27T02:18:18.963 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:18.722Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.151:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.151:8765: connect: connection refused" 2024-04-27T02:18:18.963 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:18.723Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.151:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.151:8765: connect: connection refused" 2024-04-27T02:18:19.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:18 smithi026 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:19.244 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:18 smithi026 ceph-mon[26568]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:19.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:18 smithi151 ceph-mon[22663]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:20.231 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:19 smithi026 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:20.231 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:19 smithi026 ceph-mon[26568]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:20.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:19 smithi151 ceph-mon[22663]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:21.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:21 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:21.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:21 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:21.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:21 smithi151 ceph-mon[22663]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:18:21.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:21 smithi151 ceph-mon[22663]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:18:21.584 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2743673133' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:18:21.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:21.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:21.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:18:21.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:18:21.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2743673133' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:18:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[26568]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:18:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[26568]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:18:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:21 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2743673133' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:18:22.712 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:22 smithi151 ceph-mon[22663]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:22.712 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:22 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:22.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:22 smithi026 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:22.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:22 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:22.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:22 smithi026 ceph-mon[26568]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:22.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:22 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 systemd[1]: Stopping Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 bash[51830]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-27T02:18:24.097 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-27T02:18:24.098 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-27T02:18:24.098 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-27T02:18:24.098 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-27T02:18:24.098 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[46170]: ts=2024-04-27T02:18:23.962Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-27T02:18:24.098 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:23 smithi151 podman[51843]: 2024-04-27 02:18:23.972424819 +0000 UTC m=+0.139389369 container died fcc967e841465452eb134c9881d72362c55d714431559b1713621018f6ac217c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:18:24.505 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 podman[51843]: 2024-04-27 02:18:24.294690122 +0000 UTC m=+0.461654667 container cleanup fcc967e841465452eb134c9881d72362c55d714431559b1713621018f6ac217c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:18:24.505 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 bash[51843]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a 2024-04-27T02:18:24.506 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 bash[51869]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:18:24.765 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 podman[51857]: 2024-04-27 02:18:24.529410849 +0000 UTC m=+0.557776642 container remove fcc967e841465452eb134c9881d72362c55d714431559b1713621018f6ac217c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:18:24.765 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-27T02:18:24.765 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 systemd[1]: Stopped Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:18:24.765 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.928s CPU time. 2024-04-27T02:18:24.765 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 systemd[1]: Starting Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:18:24.765 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 podman[51940]: 2024-04-27 02:18:24.669793555 +0000 UTC m=+0.019316605 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-27T02:18:24.765 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:24 smithi151 podman[51940]: 2024-04-27 02:18:24.788740164 +0000 UTC m=+0.138263195 container create 6b29e17001d04b6f7abd2b1f36d31c08f81ce8f32167f80d73aa29e01474d42a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:18:25.015 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:24 smithi151 ceph-mon[22663]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:25.015 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 podman[51940]: 2024-04-27 02:18:25.007406214 +0000 UTC m=+0.356929245 container init 6b29e17001d04b6f7abd2b1f36d31c08f81ce8f32167f80d73aa29e01474d42a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:18:25.015 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 podman[51940]: 2024-04-27 02:18:25.010133008 +0000 UTC m=+0.359656050 container start 6b29e17001d04b6f7abd2b1f36d31c08f81ce8f32167f80d73aa29e01474d42a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:18:25.015 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.033Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.033Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.033Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.033Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi151 (none))" 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.033Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.033Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.034Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.035Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.037Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.038Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.038Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.716µs 2024-04-27T02:18:25.016 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.038Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-27T02:18:25.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:24 smithi026 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:25.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:24 smithi026 ceph-mon[26568]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:18:25.201 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:25] ENGINE Bus STOPPING 2024-04-27T02:18:25.201 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:18:25.201 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:25] ENGINE Bus STOPPED 2024-04-27T02:18:25.201 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:25] ENGINE Bus STARTING 2024-04-27T02:18:25.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 bash[51940]: 6b29e17001d04b6f7abd2b1f36d31c08f81ce8f32167f80d73aa29e01474d42a 2024-04-27T02:18:25.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 systemd[1]: Started Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:18:25.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.232Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-27T02:18:25.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:25] ENGINE Serving on http://:::9283 2024-04-27T02:18:25.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:18:25] ENGINE Bus STARTED 2024-04-27T02:18:25.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.483Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.536Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.537Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.537Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.246µs wal_replay_duration=498.340503ms total_replay_duration=498.391739ms 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.540Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.540Z caller=main.go:947 level=info msg="TSDB started" 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.540Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.567Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.809817ms db_storage=741ns remote_storage=1.38µs web_handler=760ns query_engine=749ns scrape=319.468µs scrape_sd=90.384µs notify=10.948µs notify_sd=9.16µs rules=25.777131ms 2024-04-27T02:18:25.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:18:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:18:25.567Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:18:26.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:26 smithi151 ceph-mon[22663]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 869 B/s rd, 1 op/s 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:18:26.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[26568]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 869 B/s rd, 1 op/s 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:18:26.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:18:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:18:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:18:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:18:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:18:26.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:26 smithi026 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 869 B/s rd, 1 op/s 2024-04-27T02:18:29.243 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:28 smithi026 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:18:29.243 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:28 smithi026 ceph-mon[26568]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:18:29.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:28 smithi151 ceph-mon[22663]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:18:30.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:18:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-04-27T02:18:30.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:18:30.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-04-27T02:18:30.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:18:31.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:31 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:31.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:31 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:31.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:31 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:32.590 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:18:32] "GET /metrics HTTP/1.1" 200 37978 "" "Prometheus/2.33.4" 2024-04-27T02:18:32.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:32 smithi151 ceph-mon[22663]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-04-27T02:18:32.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:32 smithi026 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-04-27T02:18:32.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:32 smithi026 ceph-mon[26568]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-04-27T02:18:35.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:34 smithi151 ceph-mon[22663]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:35.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:34 smithi026 ceph-mon[21867]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:35.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:34 smithi026 ceph-mon[26568]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:37.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:36 smithi151 ceph-mon[22663]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:37.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:36 smithi026 ceph-mon[21867]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:37.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:36 smithi026 ceph-mon[26568]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:38.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:37 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:38.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:37 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:38.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:37 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:39.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:38 smithi151 ceph-mon[22663]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:39.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:38 smithi026 ceph-mon[21867]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:39.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:38 smithi026 ceph-mon[26568]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:18:40.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:40 smithi151 ceph-mon[22663]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:18:40.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:40 smithi026 ceph-mon[21867]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:18:40.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:40 smithi026 ceph-mon[26568]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:18:41.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:41 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:41.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:41 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:41.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:41 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:42.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:42 smithi151 ceph-mon[22663]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:42.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:42 smithi026 ceph-mon[21867]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:42.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:18:42] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-27T02:18:42.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:42 smithi026 ceph-mon[26568]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:45.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:44 smithi151 ceph-mon[22663]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:45.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:44 smithi026 ceph-mon[21867]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:45.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:44 smithi026 ceph-mon[26568]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:47.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:46 smithi151 ceph-mon[22663]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:47.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:46 smithi026 ceph-mon[21867]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:47.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:46 smithi026 ceph-mon[26568]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:49.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:48 smithi151 ceph-mon[22663]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:49.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:48 smithi026 ceph-mon[21867]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:49.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:48 smithi026 ceph-mon[26568]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:51.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:50 smithi151 ceph-mon[22663]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:51.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:50 smithi026 ceph-mon[21867]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:51.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:50 smithi026 ceph-mon[26568]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:52.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:51 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:52.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:51 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:52.257 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:18:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:18:52] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-27T02:18:52.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:51 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:18:53.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:52 smithi151 ceph-mon[22663]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:53.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:52 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:53.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:52 smithi026 ceph-mon[21867]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:53.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:52 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:53.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:52 smithi026 ceph-mon[26568]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:53.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:52 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:18:54.247 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:53 smithi151 ceph-mon[22663]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:54.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:53 smithi026 ceph-mon[21867]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:54.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:53 smithi026 ceph-mon[26568]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:18:57.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:56 smithi151 ceph-mon[22663]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:57.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:56 smithi026 ceph-mon[21867]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:57.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:56 smithi026 ceph-mon[26568]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:59.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:18:58 smithi151 ceph-mon[22663]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:59.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:18:58 smithi026 ceph-mon[21867]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:18:59.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:18:58 smithi026 ceph-mon[26568]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:00 smithi151 ceph-mon[22663]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:01.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:00 smithi026 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:01.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:00 smithi026 ceph-mon[26568]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:02.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:01 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:02.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:01 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:02.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:01 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:02.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:19:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:19:02] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-27T02:19:03.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:02 smithi151 ceph-mon[22663]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:03.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:02 smithi026 ceph-mon[21867]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:03.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:02 smithi026 ceph-mon[26568]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:04.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:04 smithi151 ceph-mon[22663]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:04.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:04 smithi026 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:04.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:04 smithi026 ceph-mon[26568]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:07.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:06 smithi151 ceph-mon[22663]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:07.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:06 smithi026 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:07.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:06 smithi026 ceph-mon[26568]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:08.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:07 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:08.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:07 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:08.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:07 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:09.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:08 smithi151 ceph-mon[22663]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:09.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:08 smithi026 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:09.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:08 smithi026 ceph-mon[26568]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:11.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:10 smithi151 ceph-mon[22663]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:11.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:10 smithi026 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:11.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:10 smithi026 ceph-mon[26568]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:12.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:11 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:12.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:11 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:12.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:11 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:12.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:19:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:19:12] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-27T02:19:13.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:12 smithi151 ceph-mon[22663]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:13.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:12 smithi026 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:13.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:12 smithi026 ceph-mon[26568]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:14.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:14 smithi151 ceph-mon[22663]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:14.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:14 smithi026 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:14.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:14 smithi026 ceph-mon[26568]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:16 smithi151 ceph-mon[22663]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:17.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:16 smithi026 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:17.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:16 smithi026 ceph-mon[26568]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:19.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:18 smithi151 ceph-mon[22663]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:19.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:18 smithi026 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:19.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:18 smithi026 ceph-mon[26568]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-04-27T02:19:21.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:20 smithi151 ceph-mon[22663]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:21.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:20 smithi026 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:21.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:20 smithi026 ceph-mon[26568]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:22.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:21 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:22.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:21 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:22.257 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:19:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:19:22] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-27T02:19:22.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:21 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:23.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:22 smithi151 ceph-mon[22663]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:23.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:22 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:23.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:22 smithi026 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:23.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:22 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:23.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:22 smithi026 ceph-mon[26568]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:23.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:22 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:24.247 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:24 smithi151 ceph-mon[22663]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:24.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:23 smithi026 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:24.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:23 smithi026 ceph-mon[26568]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:27.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:26 smithi151 ceph-mon[22663]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:27.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:26 smithi026 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:27.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:26 smithi026 ceph-mon[26568]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:29.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:28 smithi151 ceph-mon[22663]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:29.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:28 smithi026 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:28 smithi026 ceph-mon[26568]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:30.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:29 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:19:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:29 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:19:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:29 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:19:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:19:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:19:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:30 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:19:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:19:31.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:30 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:19:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:31 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:31 smithi151 ceph-mon[22663]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-27T02:19:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:31 smithi151 ceph-mon[22663]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:32.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:31 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:32.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:31 smithi026 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-27T02:19:32.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:31 smithi026 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:32.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:19:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:19:32] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-04-27T02:19:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:31 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:31 smithi026 ceph-mon[26568]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-27T02:19:32.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:31 smithi026 ceph-mon[26568]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:34.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:33 smithi151 ceph-mon[22663]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-27T02:19:34.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:33 smithi026 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-27T02:19:34.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:33 smithi026 ceph-mon[26568]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-27T02:19:36.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:35 smithi151 ceph-mon[22663]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:36.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:35 smithi026 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:36.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:35 smithi026 ceph-mon[26568]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:37.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:37 smithi151 ceph-mon[22663]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:37.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:37 smithi026 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:37.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:37 smithi026 ceph-mon[26568]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:38.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:38 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:38.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:38 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:38.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:38 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:39.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:39 smithi151 ceph-mon[22663]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:39.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:39 smithi026 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:39.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:39 smithi026 ceph-mon[26568]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:19:41.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:41 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:41.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:41 smithi151 ceph-mon[22663]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:19:41.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:41 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:41.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:41 smithi026 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:19:41.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:41 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:41.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:41 smithi026 ceph-mon[26568]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:19:42.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:19:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:19:42] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-27T02:19:44.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:43 smithi151 ceph-mon[22663]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:44.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:43 smithi026 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:44.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:43 smithi026 ceph-mon[26568]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:46.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:45 smithi151 ceph-mon[22663]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:46.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:45 smithi026 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:46.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:45 smithi026 ceph-mon[26568]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:48.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:48 smithi151 ceph-mon[22663]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:48 smithi026 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:48.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:48 smithi026 ceph-mon[26568]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:49.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:49 smithi151 ceph-mon[22663]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:49.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:49 smithi026 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:49.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:49 smithi026 ceph-mon[26568]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:51.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:51 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:51.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:51 smithi151 ceph-mon[22663]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:51.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:51 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:51.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:51 smithi026 ceph-mon[26568]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:51.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:51 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:19:51.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:51 smithi026 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:52.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:19:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:19:52] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-27T02:19:52.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:52 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:52 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:52.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:52 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:19:54.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:53 smithi151 ceph-mon[22663]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:54.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:53 smithi026 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:54.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:53 smithi026 ceph-mon[26568]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:56.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:55 smithi151 ceph-mon[22663]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:56.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:55 smithi026 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:56.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:55 smithi026 ceph-mon[26568]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:19:57.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:57 smithi151 ceph-mon[22663]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:57.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:57 smithi026 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:19:57.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:57 smithi026 ceph-mon[26568]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:00.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:19:59 smithi151 ceph-mon[22663]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:00.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:19:59 smithi026 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:19:59 smithi026 ceph-mon[26568]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:01 smithi151 ceph-mon[22663]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-27T02:20:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:01 smithi151 ceph-mon[22663]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-27T02:20:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:01 smithi151 ceph-mon[22663]: daemon grafana.a on smithi151 is in error state 2024-04-27T02:20:01.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:01 smithi026 ceph-mon[21867]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-27T02:20:01.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:01 smithi026 ceph-mon[21867]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-27T02:20:01.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:01 smithi026 ceph-mon[21867]: daemon grafana.a on smithi151 is in error state 2024-04-27T02:20:01.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:01 smithi026 ceph-mon[26568]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-27T02:20:01.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:01 smithi026 ceph-mon[26568]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-27T02:20:01.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:01 smithi026 ceph-mon[26568]: daemon grafana.a on smithi151 is in error state 2024-04-27T02:20:02.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:02 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:02.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:02 smithi151 ceph-mon[22663]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:02.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:02 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:02.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:02 smithi026 ceph-mon[26568]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:02.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:02 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:02.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:02 smithi026 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:02.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:20:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:20:02] "GET /metrics HTTP/1.1" 200 37967 "" "Prometheus/2.33.4" 2024-04-27T02:20:04.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:03 smithi151 ceph-mon[22663]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:04.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:03 smithi026 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:04.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:03 smithi026 ceph-mon[26568]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:06.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:05 smithi151 ceph-mon[22663]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:06.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:05 smithi026 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:06.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:05 smithi026 ceph-mon[26568]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:07.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:07 smithi151 ceph-mon[22663]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:07.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:07 smithi026 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:07.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:07 smithi026 ceph-mon[26568]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:08.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:08 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:08.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:08 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:08.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:08 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:09.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:09 smithi151 ceph-mon[22663]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:09.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:09 smithi026 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:09.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:09 smithi026 ceph-mon[26568]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:11.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:11 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:11.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:11 smithi151 ceph-mon[22663]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:11.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:11 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:11.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:11 smithi026 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:11.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:11 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:11.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:11 smithi026 ceph-mon[26568]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:12.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:20:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:20:12] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-04-27T02:20:14.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:13 smithi151 ceph-mon[22663]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:14.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:13 smithi026 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:14.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:13 smithi026 ceph-mon[26568]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:16.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:16 smithi151 ceph-mon[22663]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:16.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:16 smithi026 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:16.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:16 smithi026 ceph-mon[26568]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:17.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:17 smithi151 ceph-mon[22663]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:17 smithi026 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:17.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:17 smithi026 ceph-mon[26568]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:20.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:20 smithi151 ceph-mon[22663]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:20.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:20 smithi026 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:20.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:20 smithi026 ceph-mon[26568]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:21.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:21 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:21.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:21 smithi151 ceph-mon[22663]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:21.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:21 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:21.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:21 smithi026 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:21.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:21 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:21.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:21 smithi026 ceph-mon[26568]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:22.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:20:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:20:22] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-04-27T02:20:22.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:22 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:22.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:22 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:22.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:22 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:24.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:24 smithi151 ceph-mon[22663]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:24.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:24 smithi026 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:24.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:24 smithi026 ceph-mon[26568]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:26.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:26 smithi151 ceph-mon[22663]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:26.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:26 smithi026 ceph-mon[26568]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:26.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:26 smithi026 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:27.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:27 smithi151 ceph-mon[22663]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:27.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:27 smithi026 ceph-mon[26568]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:27.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:27 smithi026 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:30.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:30 smithi151 ceph-mon[22663]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:30 smithi026 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:30.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:30 smithi026 ceph-mon[26568]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:31 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:31 smithi151 ceph-mon[22663]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:31 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:31 smithi026 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:31.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:31 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:31.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:31 smithi026 ceph-mon[26568]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:32 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:20:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:32 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:20:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:32 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:20:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:32 smithi151 ceph-mon[22663]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-27T02:20:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:32 smithi151 ceph-mon[22663]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:32 smithi151 ceph-mon[22663]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:20:32.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:20:32] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[26568]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[26568]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[26568]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:20:32.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:20:32.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-27T02:20:32.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:32.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:32 smithi026 ceph-mon[21867]: from='mgr.15204 ' entity='mgr.y' 2024-04-27T02:20:35.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:35 smithi151 ceph-mon[22663]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:35.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:35 smithi026 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:35.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:35 smithi026 ceph-mon[26568]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:36.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:36 smithi151 ceph-mon[22663]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:36.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:36 smithi026 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:36.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:36 smithi026 ceph-mon[26568]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:37.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:37 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:37.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:37 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:37 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:39.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:39 smithi151 ceph-mon[22663]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:20:39.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:39 smithi026 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:20:39.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:39 smithi026 ceph-mon[26568]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:20:40.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:40 smithi151 ceph-mon[22663]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:40.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:40 smithi026 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:40.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:40 smithi026 ceph-mon[26568]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:20:41.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:41 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:41.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:41 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:41.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:41 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:42.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:42 smithi151 ceph-mon[22663]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:20:42.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:42 smithi026 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:20:42.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:42 smithi026 ceph-mon[26568]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:20:42.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:20:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:20:42] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:20:44.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:44 smithi151 ceph-mon[22663]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:44.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:44 smithi026 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:44.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:44 smithi026 ceph-mon[26568]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:47.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:46 smithi151 ceph-mon[22663]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:47.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:46 smithi026 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:47.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:46 smithi026 ceph-mon[26568]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:49.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:49 smithi151 ceph-mon[22663]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:49.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:49 smithi026 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:49.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:49 smithi026 ceph-mon[26568]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:50.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:50 smithi151 ceph-mon[22663]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:50.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:50 smithi026 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:50.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:50 smithi026 ceph-mon[26568]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:51.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:51 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:51.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:51 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:51.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:51 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:20:52.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:52 smithi151 ceph-mon[22663]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:52.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:52 smithi026 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:52.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:52 smithi026 ceph-mon[26568]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:52.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:20:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:20:52] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:20:53.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:53 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:53.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:53 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:53.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:53 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:20:54.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:54 smithi151 ceph-mon[22663]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:54.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:54 smithi026 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:54.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:54 smithi026 ceph-mon[26568]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:20:57.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:56 smithi151 ceph-mon[22663]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:57.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:56 smithi026 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:57.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:56 smithi026 ceph-mon[26568]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:20:58 smithi151 ceph-mon[22663]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:20:58 smithi026 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:20:59.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:20:58 smithi026 ceph-mon[26568]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:01.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:00 smithi151 ceph-mon[22663]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:01.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:00 smithi026 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:01.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:00 smithi026 ceph-mon[26568]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:02.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:01 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:02.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:01 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:02.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:01 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:02.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:21:02] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-27T02:21:03.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:02 smithi151 ceph-mon[22663]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:03.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:02 smithi026 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:03.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:02 smithi026 ceph-mon[26568]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:04.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:04 smithi151 ceph-mon[22663]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:04.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:04 smithi026 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:04.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:04 smithi026 ceph-mon[26568]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:07.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:06 smithi151 ceph-mon[22663]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:07.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:06 smithi026 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:07.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:06 smithi026 ceph-mon[26568]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:08.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:07 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:08.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:07 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:08.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:07 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:09.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:08 smithi151 ceph-mon[22663]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:09.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:08 smithi026 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:09.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:08 smithi026 ceph-mon[26568]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:10.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:21:10.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:10 smithi026 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:10.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:10 smithi026 ceph-mon[26568]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:11.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:10 smithi151 ceph-mon[22663]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:12.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:11 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:12.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:11 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:12.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:11 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:12.309 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:21:12] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-27T02:21:12.582 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (9m) 2m ago 17m 23.7M - 0.23.0 ba2b418f427c 1a93078168a0 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 error 2m ago 16m - - 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (2m) 2m ago 16m 50.4M - 3.5 e1d6a67b021e 706a44ff2db8 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283 running (11m) 2m ago 21m 448M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 210983b6ef19 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) 2m ago 23m 499M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (23m) 2m ago 23m 65.2M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (21m) 2m ago 21m 45.7M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (22m) 2m ago 22m 47.7M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (9m) 2m ago 17m 18.2M - 1.3.1 1dbe0e931976 b2e202622fa1 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (8m) 2m ago 17m 17.1M - 1.3.1 1dbe0e931976 00d8989305ef 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (21m) 2m ago 21m 53.5M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (20m) 2m ago 20m 54.2M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (20m) 2m ago 20m 49.9M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (20m) 2m ago 20m 56.0M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:21:12.583 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (19m) 2m ago 19m 54.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:21:12.584 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (19m) 2m ago 19m 53.8M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:21:12.584 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (19m) 2m ago 19m 49.9M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:21:12.584 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (18m) 2m ago 18m 53.8M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:21:12.584 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (2m) 2m ago 17m 44.6M - 2.33.4 514e6a882f6e 6b29e17001d0 2024-04-27T02:21:12.584 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (16m) 2m ago 16m 74.3M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:21:12.584 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (16m) 2m ago 16m 76.3M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:21:12.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:12 smithi026 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:12.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:12 smithi026 ceph-mon[26568]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:13.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:12 smithi151 ceph-mon[22663]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:13.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-27T02:21:14.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:13 smithi151 ceph-mon[22663]: from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:21:14.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:13 smithi026 ceph-mon[21867]: from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:21:14.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:13 smithi026 ceph-mon[26568]: from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:21:14.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:14 smithi026 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:14.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:14 smithi026 ceph-mon[26568]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:15.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:14 smithi151 ceph-mon[22663]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:15.488 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:21:15.488 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:21:15.489 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:21:15.490 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:21:15.490 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:21:15.490 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:21:16.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:15 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3681812120' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:21:16.062 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:15 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3681812120' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:21:16.087 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph -s' 2024-04-27T02:21:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:15 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3681812120' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:21:17.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:16 smithi026 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:17.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:16 smithi026 ceph-mon[26568]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:17.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:16 smithi151 ceph-mon[22663]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-27T02:21:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: id: 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:21:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_WARN 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: 1 failed cephadm daemon(s) 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: y(active, since 3m), standbys: x 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 241 objects, 585 KiB 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-27T02:21:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:21:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-27T02:21:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-27T02:21:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-27T02:21:18.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:18 smithi026 ceph-mon[21867]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:18.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:18 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3470597670' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:21:18.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:18 smithi026 ceph-mon[26568]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:18.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:18 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3470597670' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:21:19.008 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:21:19.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:18 smithi151 ceph-mon[22663]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:19.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:18 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3470597670' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-27T02:21:21.011 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-27T02:21:21.011 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-27T02:21:21.011 INFO:teuthology.orchestra.run.smithi026.stdout: daemon grafana.a on smithi151 is in error state 2024-04-27T02:21:21.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:20 smithi026 ceph-mon[21867]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:21.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:20 smithi026 ceph-mon[26568]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:21.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:20 smithi151 ceph-mon[22663]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:21.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-27T02:21:21.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:21 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:21.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:21 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2051042972' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:21:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:21 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:21 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2051042972' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:21:22.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:21 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2051042972' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:21:22.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:21:22] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-27T02:21:22.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:22 smithi026 ceph-mon[21867]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:22.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:22 smithi026 ceph-mon[21867]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:22.935 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:22 smithi026 ceph-mon[26568]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:22.935 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:22 smithi026 ceph-mon[26568]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:23.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:22 smithi151 ceph-mon[22663]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:23.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:22 smithi151 ceph-mon[22663]: from='mgr.15204 172.21.15.26:0/3042197330' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:23.719 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:21:24.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:24 smithi026 ceph-mon[21867]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:24.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:24 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/320231409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:21:24.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:24 smithi026 ceph-mon[26568]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:24.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:24 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/320231409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:21:24.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph mgr fail' 2024-04-27T02:21:24.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:24 smithi151 ceph-mon[22663]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:21:24.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:24 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/320231409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:21:27.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-mon[22663]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3834715914' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-mon[22663]: osdmap e103: 8 total, 8 up, 8 in 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:26] ENGINE Bus STOPPING 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:26] ENGINE Bus STOPPED 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:26] ENGINE Bus STARTING 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:26] ENGINE Serving on http://:::9283 2024-04-27T02:21:27.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:26] ENGINE Bus STARTED 2024-04-27T02:21:27.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:27.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3834715914' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:21:27.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:21:27.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[21867]: osdmap e103: 8 total, 8 up, 8 in 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[26568]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3834715914' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mon[26568]: osdmap e103: 8 total, 8 up, 8 in 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ignoring --setuser ceph since I am not root 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ignoring --setgroup ceph since I am not root 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-mgr[60490]: -- 172.21.15.26:0/322800636 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ba8a5f45a0 con 0x55ba8a5c8c00 2024-04-27T02:21:27.105 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:26.957+0000 7f8dd3271200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:21:27.465 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:27.155+0000 7f8dd3271200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:21:27.466 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:27.266+0000 7f8dd3271200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:21:27.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 180' 2024-04-27T02:21:27.745 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:27.605+0000 7f8dd3271200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:21:27.745 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:27.709+0000 7f8dd3271200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:21:27.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: mgrmap e33: x(active, starting, since 0.420131s) 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:21:27.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: Manager daemon x is now available 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:21:27.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:21:28.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:21:28.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:21:28.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: mgrmap e33: x(active, starting, since 0.420131s) 2024-04-27T02:21:28.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:21:28.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:21:28.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:21:28.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: Manager daemon x is now available 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:28.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:28.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:21:28.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:21:28.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:21:28.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:27 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:21:28.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: mgrmap e33: x(active, starting, since 0.420131s) 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:21:28.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: Manager daemon x is now available 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:21:28.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:27 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[26568]: [27/Apr/2024:02:21:27] ENGINE Bus STARTING 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[26568]: [27/Apr/2024:02:21:27] ENGINE Serving on https://172.21.15.151:7150 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[26568]: mgrmap e34: x(active, since 1.43752s) 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[26568]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[26568]: [27/Apr/2024:02:21:27] ENGINE Serving on http://172.21.15.151:8765 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[26568]: [27/Apr/2024:02:21:27] ENGINE Bus STARTED 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[21867]: [27/Apr/2024:02:21:27] ENGINE Bus STARTING 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[21867]: [27/Apr/2024:02:21:27] ENGINE Serving on https://172.21.15.151:7150 2024-04-27T02:21:29.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[21867]: mgrmap e34: x(active, since 1.43752s) 2024-04-27T02:21:29.020 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:29.020 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[21867]: [27/Apr/2024:02:21:27] ENGINE Serving on http://172.21.15.151:8765 2024-04-27T02:21:29.020 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:28 smithi026 ceph-mon[21867]: [27/Apr/2024:02:21:27] ENGINE Bus STARTED 2024-04-27T02:21:29.020 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:29.009+0000 7f8dd3271200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:21:29.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:28 smithi151 ceph-mon[22663]: [27/Apr/2024:02:21:27] ENGINE Bus STARTING 2024-04-27T02:21:29.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:28 smithi151 ceph-mon[22663]: [27/Apr/2024:02:21:27] ENGINE Serving on https://172.21.15.151:7150 2024-04-27T02:21:29.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:28 smithi151 ceph-mon[22663]: mgrmap e34: x(active, since 1.43752s) 2024-04-27T02:21:29.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:28 smithi151 ceph-mon[22663]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:29.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:28 smithi151 ceph-mon[22663]: [27/Apr/2024:02:21:27] ENGINE Serving on http://172.21.15.151:8765 2024-04-27T02:21:29.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:28 smithi151 ceph-mon[22663]: [27/Apr/2024:02:21:27] ENGINE Bus STARTED 2024-04-27T02:21:29.337 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:29.197+0000 7f8dd3271200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:21:29.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:29.378+0000 7f8dd3271200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:21:29.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:29 smithi151 ceph-mon[22663]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:29.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:29 smithi151 ceph-mon[22663]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:21:29.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:29 smithi151 ceph-mon[22663]: Cluster is now healthy 2024-04-27T02:21:29.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:29 smithi151 ceph-mon[22663]: mgrmap e35: x(active, since 2s) 2024-04-27T02:21:29.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:29.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[21867]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:21:29.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[21867]: Cluster is now healthy 2024-04-27T02:21:29.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[21867]: mgrmap e35: x(active, since 2s) 2024-04-27T02:21:29.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[26568]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:29.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[26568]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:21:29.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[26568]: Cluster is now healthy 2024-04-27T02:21:29.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-mon[26568]: mgrmap e35: x(active, since 2s) 2024-04-27T02:21:29.991 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:29.690+0000 7f8dd3271200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:21:29.991 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:29.790+0000 7f8dd3271200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:21:29.991 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:30 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:30.007+0000 7f8dd3271200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:21:30.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:30 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:30.216+0000 7f8dd3271200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:21:31.104 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:30 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:30.883+0000 7f8dd3271200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:21:31.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:31.354+0000 7f8dd3271200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:21:31.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:31.457+0000 7f8dd3271200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:21:32.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:31 smithi151 ceph-mon[22663]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:32.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:31 smithi151 ceph-mon[22663]: mgrmap e36: x(active, since 4s) 2024-04-27T02:21:32.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:31 smithi151 ceph-mon[22663]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:32.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:31 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:32.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:31 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:32.104 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:31.672+0000 7f8dd3271200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:21:32.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:32.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[21867]: mgrmap e36: x(active, since 4s) 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[21867]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[26568]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[26568]: mgrmap e36: x(active, since 4s) 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[26568]: from='client.25052 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:32.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:31 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:32.582 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:32.245+0000 7f8dd3271200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:21:32.582 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:32.344+0000 7f8dd3271200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:21:32.841 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:32.605+0000 7f8dd3271200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:21:32.841 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:32.757+0000 7f8dd3271200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:21:32.841 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:32.864+0000 7f8dd3271200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:21:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:33.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:33.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:33.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:33.183 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:32.991+0000 7f8dd3271200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:21:33.183 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:33.093+0000 7f8dd3271200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:21:33.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:32 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:33.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:32 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:34.221 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:33.986+0000 7f8dd3271200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:21:34.221 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:34.223+0000 7f8dd3271200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:21:34.221 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:21:34] ENGINE Bus STARTING 2024-04-27T02:21:34.604 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=warn ts=2024-04-27T02:21:34.245Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: Standby manager daemon y started 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: CherryPy Checker: 2024-04-27T02:21:34.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: The Application mounted at '' has an empty config. 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:21:34] ENGINE Serving on http://:::9283 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:21:34] ENGINE Bus STARTED 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: Standby manager daemon y started 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:21:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:21:34.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:21:34.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:34.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:34.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:34.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: Standby manager daemon y started 2024-04-27T02:21:34.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-27T02:21:34.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:21:34.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-27T02:21:34.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:34 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:21:35.104 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:21:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:21:34.851Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=2 2024-04-27T02:21:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:35 smithi151 ceph-mon[22663]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-27T02:21:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:35 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:21:35.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:35 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:35.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:35 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:35.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:35 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:35.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:35 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[26568]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-27T02:21:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:21:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[21867]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-27T02:21:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:21:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:35 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:21:36.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:36.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:36 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:36.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:36.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:36.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:36 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:37.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:37 smithi151 ceph-mon[22663]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-27T02:21:37.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:37 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:21:37.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:37 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:21:37.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:37 smithi151 ceph-mon[22663]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[21867]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[26568]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:21:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:37 smithi026 ceph-mon[26568]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:21:39.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:21:38.957+0000 7f72cf647700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:39.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:21:39.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:39.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:39.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:39.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:39.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:39.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:39.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:21:39.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:21:39.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:21:39.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:21:39.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return future.result(timeout) 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return self.__get_result() 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: raise self._exception 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: config_blobs=daemon_spec.final_config, 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:21:39.349 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Creating ceph-iscsi config... 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: Traceback (most recent call last): 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: return _run_code(code, main_globals, None, 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: exec(code, run_globals) 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:39.350 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:39.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:39.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:39.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:39.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:39.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:39.351 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:39.716 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:39 smithi151 ceph-mon[22663]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:21:39.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:39 smithi026 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:21:39.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:39 smithi026 ceph-mon[26568]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:40 smithi151 ceph-mon[22663]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:40.596 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:21:40.597 INFO:journalctl@ceph.mon.b.smithi151.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: return future.result(timeout) 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.__get_result() 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: raise self._exception 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:21:40.598 INFO:journalctl@ceph.mon.b.smithi151.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:40.599 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:40 smithi151 ceph-mon[22663]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:21:40.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:40 smithi151 ceph-mon[22663]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:40 smithi026 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:21:40.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:21:40.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:40.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:40.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:40 smithi026 ceph-mon[26568]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:21:40.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:21:40.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:21:40.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:21:40.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:40 smithi026 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:40 smithi026 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:21:40.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:21:40.864 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:21:40.864 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:21:40.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:40 smithi026 ceph-mon[26568]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:21:40.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:40 smithi026 ceph-mon[26568]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:21:41.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:41 smithi151 ceph-mon[22663]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:41.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:41 smithi026 ceph-mon[26568]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:41.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:41 smithi026 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:42.396 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 systemd[1]: Stopping Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:21:42.396 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 bash[54796]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:21:42.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:42 smithi026 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:42.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:42 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:42.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:42 smithi026 ceph-mon[26568]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:42.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:42 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:42.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:21:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:21:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-27T02:21:42.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-mon[22663]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:42.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.419Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.420Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[51955]: ts=2024-04-27T02:21:42.421Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-27T02:21:42.705 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 podman[54807]: 2024-04-27 02:21:42.430270727 +0000 UTC m=+0.116568100 container died 6b29e17001d04b6f7abd2b1f36d31c08f81ce8f32167f80d73aa29e01474d42a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:21:43.080 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 podman[54807]: 2024-04-27 02:21:42.828181497 +0000 UTC m=+0.514478877 container cleanup 6b29e17001d04b6f7abd2b1f36d31c08f81ce8f32167f80d73aa29e01474d42a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:21:43.080 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 bash[54807]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a 2024-04-27T02:21:43.080 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:42 smithi151 bash[54833]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:21:43.331 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 podman[54821]: 2024-04-27 02:21:43.104578193 +0000 UTC m=+0.674050289 container remove 6b29e17001d04b6f7abd2b1f36d31c08f81ce8f32167f80d73aa29e01474d42a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:21:43.391 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-27T02:21:43.391 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 systemd[1]: Stopped Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:21:43.392 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.338s CPU time. 2024-04-27T02:21:43.392 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 systemd[1]: Starting Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:21:43.392 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 podman[54907]: 2024-04-27 02:21:43.240743428 +0000 UTC m=+0.019395446 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-27T02:21:43.392 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 podman[54907]: 2024-04-27 02:21:43.355483396 +0000 UTC m=+0.134135419 container create 0d1b89e74e1423fde785bab62bfca5f03a9f4c92dfcbb779c2b34218b11a15cd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:21:43.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:43] ENGINE Bus STOPPING 2024-04-27T02:21:43.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:21:43.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:43] ENGINE Bus STOPPED 2024-04-27T02:21:43.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:43] ENGINE Bus STARTING 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 podman[54907]: 2024-04-27 02:21:43.624411012 +0000 UTC m=+0.403063030 container init 0d1b89e74e1423fde785bab62bfca5f03a9f4c92dfcbb779c2b34218b11a15cd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 podman[54907]: 2024-04-27 02:21:43.627025669 +0000 UTC m=+0.405677690 container start 0d1b89e74e1423fde785bab62bfca5f03a9f4c92dfcbb779c2b34218b11a15cd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.650Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.650Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.650Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.650Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi151 (none))" 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.650Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.650Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.651Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.652Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.653Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.654Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.654Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.811µs 2024-04-27T02:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.654Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-27T02:21:43.855 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 bash[54907]: 0d1b89e74e1423fde785bab62bfca5f03a9f4c92dfcbb779c2b34218b11a15cd 2024-04-27T02:21:43.855 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 systemd[1]: Started Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:21:43.855 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:43.877Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-04-27T02:21:44.144 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.122Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-04-27T02:21:44.144 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:43] ENGINE Serving on http://:::9283 2024-04-27T02:21:44.144 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:21:43] ENGINE Bus STARTED 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.168Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.217Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.218Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.218Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=34.462µs wal_replay_duration=563.44528ms total_replay_duration=563.4922ms 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.221Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.221Z caller=main.go:947 level=info msg="TSDB started" 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.221Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-27T02:21:44.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.242Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.355271ms db_storage=802ns remote_storage=1.245µs web_handler=629ns query_engine=613ns scrape=547.105µs scrape_sd=87.213µs notify=9.793µs notify_sd=12.657µs rules=19.391913ms 2024-04-27T02:21:44.597 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:21:44.242Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:44.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:44 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:45.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:45.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:45.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:44 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:47.064 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:46 smithi151 ceph-mon[22663]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-27T02:21:47.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:46 smithi026 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-27T02:21:47.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:46 smithi026 ceph-mon[26568]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-27T02:21:48.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:48.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 512 B/s rd, 0 op/s 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:48.677 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 512 B/s rd, 0 op/s 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:48.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:48 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 512 B/s rd, 0 op/s 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:49.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:48 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:21:50.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:50 smithi026 ceph-mon[21867]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:50.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:50 smithi026 ceph-mon[26568]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:51.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:50 smithi151 ceph-mon[22663]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:53.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:53.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:53.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:53.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:21:53.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1557405037' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3119489274' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:21:53.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:21:53.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:21:53.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:21:53.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:53.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1557405037' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:21:53.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3119489274' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]: dispatch 2024-04-27T02:21:53.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:53 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]: dispatch 2024-04-27T02:21:53.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:53.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:53.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:53.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:21:53.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:21:53.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:21:53.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:21:53.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:21:53.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1557405037' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:21:53.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3119489274' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]: dispatch 2024-04-27T02:21:53.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:53 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]: dispatch 2024-04-27T02:21:54.497 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:21:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:21:54] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-27T02:21:55.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[21867]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-27T02:21:55.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]': finished 2024-04-27T02:21:55.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[21867]: osdmap e104: 8 total, 8 up, 8 in 2024-04-27T02:21:55.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2358337284' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1803422907"}]: dispatch 2024-04-27T02:21:55.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[26568]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-27T02:21:55.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[26568]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]': finished 2024-04-27T02:21:55.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[26568]: osdmap e104: 8 total, 8 up, 8 in 2024-04-27T02:21:55.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:55 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2358337284' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1803422907"}]: dispatch 2024-04-27T02:21:55.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:55 smithi151 ceph-mon[22663]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-27T02:21:55.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:55 smithi151 ceph-mon[22663]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4219469282"}]': finished 2024-04-27T02:21:55.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:55 smithi151 ceph-mon[22663]: osdmap e104: 8 total, 8 up, 8 in 2024-04-27T02:21:55.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:55 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2358337284' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1803422907"}]: dispatch 2024-04-27T02:21:56.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:56 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2358337284' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1803422907"}]': finished 2024-04-27T02:21:56.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:56 smithi026 ceph-mon[26568]: osdmap e105: 8 total, 8 up, 8 in 2024-04-27T02:21:56.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:56 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4273736426' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3576165429"}]: dispatch 2024-04-27T02:21:56.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:56 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2358337284' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1803422907"}]': finished 2024-04-27T02:21:56.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:56 smithi026 ceph-mon[21867]: osdmap e105: 8 total, 8 up, 8 in 2024-04-27T02:21:56.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:56 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4273736426' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3576165429"}]: dispatch 2024-04-27T02:21:56.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:56 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2358337284' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1803422907"}]': finished 2024-04-27T02:21:56.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:56 smithi151 ceph-mon[22663]: osdmap e105: 8 total, 8 up, 8 in 2024-04-27T02:21:56.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:56 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4273736426' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3576165429"}]: dispatch 2024-04-27T02:21:57.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-27T02:21:57.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/4273736426' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3576165429"}]': finished 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: osdmap e106: 8 total, 8 up, 8 in 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/137420494' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2085984642"}]: dispatch 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:57 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-27T02:21:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/4273736426' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3576165429"}]': finished 2024-04-27T02:21:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: osdmap e106: 8 total, 8 up, 8 in 2024-04-27T02:21:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/137420494' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2085984642"}]: dispatch 2024-04-27T02:21:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/4273736426' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3576165429"}]': finished 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: osdmap e106: 8 total, 8 up, 8 in 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/137420494' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2085984642"}]: dispatch 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:21:57.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:21:57.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:21:57.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:57 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:21:58.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:58 smithi151 ceph-mon[22663]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-27T02:21:58.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:58 smithi151 ceph-mon[22663]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:58.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:58 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/137420494' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2085984642"}]': finished 2024-04-27T02:21:58.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:58 smithi151 ceph-mon[22663]: osdmap e107: 8 total, 8 up, 8 in 2024-04-27T02:21:58.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:58 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/366870998' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2085984642"}]: dispatch 2024-04-27T02:21:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[21867]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-27T02:21:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[21867]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/137420494' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2085984642"}]': finished 2024-04-27T02:21:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[21867]: osdmap e107: 8 total, 8 up, 8 in 2024-04-27T02:21:58.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/366870998' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2085984642"}]: dispatch 2024-04-27T02:21:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[26568]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-27T02:21:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[26568]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:21:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/137420494' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2085984642"}]': finished 2024-04-27T02:21:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[26568]: osdmap e107: 8 total, 8 up, 8 in 2024-04-27T02:21:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:58 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/366870998' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2085984642"}]: dispatch 2024-04-27T02:21:59.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:59 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/366870998' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2085984642"}]': finished 2024-04-27T02:21:59.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:59 smithi151 ceph-mon[22663]: osdmap e108: 8 total, 8 up, 8 in 2024-04-27T02:21:59.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:21:59 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3056158768' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1479323410"}]: dispatch 2024-04-27T02:21:59.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:59 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/366870998' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2085984642"}]': finished 2024-04-27T02:21:59.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:59 smithi026 ceph-mon[26568]: osdmap e108: 8 total, 8 up, 8 in 2024-04-27T02:21:59.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:21:59 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3056158768' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1479323410"}]: dispatch 2024-04-27T02:21:59.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:59 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/366870998' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2085984642"}]': finished 2024-04-27T02:21:59.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:59 smithi026 ceph-mon[21867]: osdmap e108: 8 total, 8 up, 8 in 2024-04-27T02:21:59.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:21:59 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3056158768' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1479323410"}]: dispatch 2024-04-27T02:22:00.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:00 smithi151 ceph-mon[22663]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-27T02:22:00.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:00 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3056158768' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1479323410"}]': finished 2024-04-27T02:22:00.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:00 smithi151 ceph-mon[22663]: osdmap e109: 8 total, 8 up, 8 in 2024-04-27T02:22:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:00 smithi026 ceph-mon[26568]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-27T02:22:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:00 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3056158768' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1479323410"}]': finished 2024-04-27T02:22:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:00 smithi026 ceph-mon[26568]: osdmap e109: 8 total, 8 up, 8 in 2024-04-27T02:22:00.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:00 smithi026 ceph-mon[21867]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-27T02:22:00.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:00 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3056158768' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1479323410"}]': finished 2024-04-27T02:22:00.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:00 smithi026 ceph-mon[21867]: osdmap e109: 8 total, 8 up, 8 in 2024-04-27T02:22:02.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:02 smithi151 ceph-mon[22663]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:02.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:02 smithi026 ceph-mon[21867]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:02.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:02 smithi026 ceph-mon[26568]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:03.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:03 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:03.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:03 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:03.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:03 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:04.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:04 smithi151 ceph-mon[22663]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:04.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:22:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:22:03] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-27T02:22:04.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:04 smithi026 ceph-mon[21867]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:04.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:04 smithi026 ceph-mon[26568]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:06.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:06 smithi151 ceph-mon[22663]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:22:06.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:06 smithi026 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:22:06.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:06 smithi026 ceph-mon[26568]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:22:08.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:08 smithi151 ceph-mon[22663]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-04-27T02:22:08.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:08 smithi026 ceph-mon[21867]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-04-27T02:22:08.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:08 smithi026 ceph-mon[26568]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-04-27T02:22:10.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:10 smithi151 ceph-mon[22663]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:22:10.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:10 smithi026 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:22:10.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:10 smithi026 ceph-mon[26568]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:22:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:12 smithi151 ceph-mon[22663]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 891 B/s rd, 0 op/s 2024-04-27T02:22:12.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:12 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:12.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:12 smithi026 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 891 B/s rd, 0 op/s 2024-04-27T02:22:12.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:12 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:12.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:12 smithi026 ceph-mon[26568]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 891 B/s rd, 0 op/s 2024-04-27T02:22:12.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:12 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:13.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:13 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:13.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:13 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:13.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:13 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:14.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:14 smithi151 ceph-mon[22663]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:14.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:22:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:22:13] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:22:14.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:14 smithi026 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:14 smithi026 ceph-mon[26568]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:16.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:16 smithi151 ceph-mon[22663]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:16.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:16 smithi026 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:16.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:16 smithi026 ceph-mon[26568]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:18.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:18 smithi151 ceph-mon[22663]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:18.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:18 smithi026 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:18.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:18 smithi026 ceph-mon[26568]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:20.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:20 smithi151 ceph-mon[22663]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:20.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:20 smithi026 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:20.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:20 smithi026 ceph-mon[26568]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:22.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:22 smithi151 ceph-mon[22663]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:22.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:22 smithi026 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:22.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:22 smithi026 ceph-mon[26568]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:23.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:23 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:23.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:23 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:23.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:23 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:24.218 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:22:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:22:23] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:22:24.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:24 smithi151 ceph-mon[22663]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:24.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:24 smithi026 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:24.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:24 smithi026 ceph-mon[26568]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:26.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:26 smithi151 ceph-mon[22663]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:26.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:26 smithi026 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:26.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:26 smithi026 ceph-mon[26568]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:27.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:27.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:27.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:28.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:28 smithi151 ceph-mon[22663]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:28.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:28 smithi026 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:28.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:28 smithi026 ceph-mon[26568]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:30.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:30 smithi151 ceph-mon[22663]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:30.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:30 smithi026 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:30.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:30 smithi026 ceph-mon[26568]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:32.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:32 smithi151 ceph-mon[22663]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:32.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:32 smithi026 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:32.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:32 smithi026 ceph-mon[26568]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:33.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:33 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:33.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:33 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:33.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:33 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:34.240 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:22:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:22:33] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-04-27T02:22:34.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:34 smithi151 ceph-mon[22663]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:34.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:34 smithi026 ceph-mon[26568]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:34.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:34 smithi026 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:36.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:36 smithi151 ceph-mon[22663]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:36.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:36 smithi026 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:36.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:36 smithi026 ceph-mon[26568]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:38.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:38 smithi151 ceph-mon[22663]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:38.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:38 smithi026 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:38.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:38 smithi026 ceph-mon[26568]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:40.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:40 smithi151 ceph-mon[22663]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:40.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:40 smithi026 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:40.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:40 smithi026 ceph-mon[26568]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:42.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:42 smithi151 ceph-mon[22663]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:42.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:42 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:42.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:42 smithi026 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:42.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:42 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:42.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:42 smithi026 ceph-mon[26568]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:42.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:42 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:43.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:43 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:43.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:43 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:43.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:43 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:44.258 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:22:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:22:43] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-27T02:22:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:44 smithi151 ceph-mon[22663]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:44 smithi026 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:44.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:44 smithi026 ceph-mon[26568]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:46.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:46 smithi151 ceph-mon[22663]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:46.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:46 smithi026 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:46.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:46 smithi026 ceph-mon[26568]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:48.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:48 smithi151 ceph-mon[22663]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:48.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:48 smithi026 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:48.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:48 smithi026 ceph-mon[26568]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:50.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:50 smithi151 ceph-mon[22663]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:50.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:50 smithi026 ceph-mon[26568]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:50.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:50 smithi026 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:52.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:52 smithi151 ceph-mon[22663]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:52.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:52 smithi026 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:52.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:52 smithi026 ceph-mon[26568]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:53.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:53 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:53.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:53 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:53.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:53 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:22:54.276 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:22:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:22:53] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-27T02:22:54.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:54 smithi151 ceph-mon[22663]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:54.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:54 smithi026 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:54.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:54 smithi026 ceph-mon[26568]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:56.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:56 smithi151 ceph-mon[22663]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:56.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:56 smithi026 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:56 smithi026 ceph-mon[26568]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:22:57.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:57 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:22:57.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:57 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:57 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:22:57.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:57 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:57.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:57 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:22:57.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:57 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:22:58.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:58 smithi151 ceph-mon[22663]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:58.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:58 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:22:58.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:58 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:22:58.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:58 smithi151 ceph-mon[22663]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-27T02:22:58.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:58 smithi151 ceph-mon[22663]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:22:58.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:22:58 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:22:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:22:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:22:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-27T02:22:58.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:22:58.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:22:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[26568]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:22:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:22:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:22:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[26568]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-27T02:22:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[26568]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:22:58.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:22:58 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:23:00.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:00 smithi151 ceph-mon[22663]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:00.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:00 smithi026 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:00.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:00 smithi026 ceph-mon[26568]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:02.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:02 smithi151 ceph-mon[22663]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:02 smithi026 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:02.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:02 smithi026 ceph-mon[26568]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:03.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:03 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:03.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:03 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:03.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:03 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:04.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:23:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:23:03] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-27T02:23:04.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:04 smithi151 ceph-mon[22663]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:04 smithi026 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:04 smithi026 ceph-mon[26568]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:06.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:06 smithi151 ceph-mon[22663]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:06.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:06 smithi026 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:06.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:06 smithi026 ceph-mon[26568]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:07.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:07 smithi151 ceph-mon[22663]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:23:07.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:07 smithi026 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:23:07.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:07 smithi026 ceph-mon[26568]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:23:10.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:10 smithi151 ceph-mon[22663]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:10.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:10 smithi026 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:10.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:10 smithi026 ceph-mon[26568]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:12.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:12 smithi151 ceph-mon[22663]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:12.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:12 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:12.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:12 smithi026 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:12.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:12 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:12 smithi026 ceph-mon[26568]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:12 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:13.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:13 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:13.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:13 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:13.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:13 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:14.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:23:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:23:13] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:23:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:14 smithi151 ceph-mon[22663]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:14.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:14 smithi026 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:14.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:14 smithi026 ceph-mon[26568]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:16.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:16 smithi151 ceph-mon[22663]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:16.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:16 smithi026 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:16.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:16 smithi026 ceph-mon[26568]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:17.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:17 smithi151 ceph-mon[22663]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:17.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:17 smithi026 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:17.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:17 smithi026 ceph-mon[26568]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:20.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:20 smithi151 ceph-mon[22663]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:20.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:20 smithi026 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:20.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:20 smithi026 ceph-mon[26568]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:22.655 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:22 smithi026 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:22.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:22 smithi026 ceph-mon[26568]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:22.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:22 smithi151 ceph-mon[22663]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:23.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:23 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:23.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:23 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:23.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:23 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:24.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:23:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:23:23] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:23:24.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:24 smithi151 ceph-mon[22663]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:24.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:24 smithi026 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:24.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:24 smithi026 ceph-mon[26568]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:25.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:25 smithi151 ceph-mon[22663]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:25 smithi026 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:25.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:25 smithi026 ceph-mon[26568]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:27.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:26 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:27.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:26 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:27.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:26 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:28.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:27 smithi151 ceph-mon[22663]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:28.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:27 smithi026 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:28.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:27 smithi026 ceph-mon[26568]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:30.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:30 smithi151 ceph-mon[22663]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:30.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:30 smithi026 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:30.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:30 smithi026 ceph-mon[26568]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:32.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:32 smithi151 ceph-mon[22663]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:32.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:32 smithi026 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:32.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:32 smithi026 ceph-mon[26568]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:33.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:33 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:33.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:33 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:33.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:33 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:34.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:23:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:23:33] "GET /metrics HTTP/1.1" 200 37979 "" "Prometheus/2.33.4" 2024-04-27T02:23:34.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:34 smithi151 ceph-mon[22663]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:34.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:34 smithi026 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:34.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:34 smithi026 ceph-mon[26568]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:35 smithi151 ceph-mon[22663]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:35.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:35 smithi026 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:35 smithi026 ceph-mon[26568]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:38.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:38 smithi151 ceph-mon[22663]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:38.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:38 smithi026 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:38.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:38 smithi026 ceph-mon[26568]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:40.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:40 smithi151 ceph-mon[22663]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:40.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:40 smithi026 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:40.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:40 smithi026 ceph-mon[26568]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:41.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:41 smithi151 ceph-mon[22663]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:41.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:41 smithi026 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:41.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:41 smithi026 ceph-mon[26568]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:42.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:42 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:42.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:42 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:42.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:42 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:42.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:42 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:42.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:42 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:42.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:42 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:44.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:44 smithi151 ceph-mon[22663]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:44.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:23:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:23:43] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-04-27T02:23:44.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:44 smithi026 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:44.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:44 smithi026 ceph-mon[26568]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:46.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:46 smithi151 ceph-mon[22663]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:46.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:46 smithi026 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:46.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:46 smithi026 ceph-mon[26568]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:47.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:47 smithi151 ceph-mon[22663]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:47.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:47 smithi026 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:47.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:47 smithi026 ceph-mon[26568]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:50.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:50 smithi151 ceph-mon[22663]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:50.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:50 smithi026 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:50.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:50 smithi026 ceph-mon[26568]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:51.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:51 smithi151 ceph-mon[22663]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:51.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:51 smithi026 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:51.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:51 smithi026 ceph-mon[26568]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:52.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:52 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:52.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:52 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:52.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:52 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:23:54.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:54 smithi151 ceph-mon[22663]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:54.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:23:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:23:53] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-04-27T02:23:54.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:54 smithi026 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:54.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:54 smithi026 ceph-mon[26568]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:23:55.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:55 smithi151 ceph-mon[22663]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:55.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:55 smithi026 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:55.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:55 smithi026 ceph-mon[26568]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:57.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:56 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:57.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:56 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:57.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:56 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:23:58.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:57 smithi151 ceph-mon[22663]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:58.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:57 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:23:58.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:57 smithi026 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:58.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:57 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:23:58.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:57 smithi026 ceph-mon[26568]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:23:58.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:57 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:23:59.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:59 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:23:59.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:59 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:23:59.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:59 smithi151 ceph-mon[22663]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:23:59.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:59 smithi151 ceph-mon[22663]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:59.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:23:59 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:23:59.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:23:59.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:23:59.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:23:59.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:59.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:23:59.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:23:59.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:23:59.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[26568]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:23:59.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[26568]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:23:59.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:23:59 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:01.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:01 smithi151 ceph-mon[22663]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:01.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:01 smithi026 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:01.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:01 smithi026 ceph-mon[26568]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:03.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:03 smithi151 ceph-mon[22663]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:24:03.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:03 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:03.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:03 smithi026 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:24:03.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:03 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:03.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:03 smithi026 ceph-mon[26568]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-04-27T02:24:03.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:03 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:04.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:03 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:24:03] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.33.4" 2024-04-27T02:24:05.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:05 smithi151 ceph-mon[22663]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:05 smithi026 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:05.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:05 smithi026 ceph-mon[26568]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:07.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:07 smithi151 ceph-mon[22663]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:07.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:07 smithi026 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:07.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:07 smithi026 ceph-mon[26568]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:24:09.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:09 smithi151 ceph-mon[22663]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:24:09.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:09 smithi026 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:24:09.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:09 smithi026 ceph-mon[26568]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:24:11.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:11 smithi151 ceph-mon[22663]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:11.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:11 smithi026 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:11.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:11 smithi026 ceph-mon[26568]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:12 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:12.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:12 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:12.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:12 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:13.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:13 smithi151 ceph-mon[22663]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:13.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:13 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:13.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:13 smithi026 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:13.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:13 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:13.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:13 smithi026 ceph-mon[26568]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:13.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:13 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:14.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:24:13] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.33.4" 2024-04-27T02:24:15.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:15 smithi151 ceph-mon[22663]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:15.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:15 smithi026 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:15.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:15 smithi026 ceph-mon[26568]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:17 smithi151 ceph-mon[22663]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:17 smithi026 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:17.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:17 smithi026 ceph-mon[26568]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:19.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:19 smithi151 ceph-mon[22663]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:19.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:19 smithi026 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:19.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:19 smithi026 ceph-mon[26568]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:21.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:21 smithi151 ceph-mon[22663]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:21.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:21 smithi026 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:21.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:21 smithi026 ceph-mon[26568]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:23.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:23 smithi151 ceph-mon[22663]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:23.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:23 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:23.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:23 smithi026 ceph-mon[21867]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:23.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:23 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:23.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:23 smithi026 ceph-mon[26568]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:23.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:23 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:24.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:24:23] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.33.4" 2024-04-27T02:24:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:25 smithi151 ceph-mon[22663]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:25.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:25 smithi026 ceph-mon[21867]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:25.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:25 smithi026 ceph-mon[26568]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:27.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:27 smithi151 ceph-mon[22663]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:27.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:27 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:27.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:27 smithi026 ceph-mon[21867]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:27.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:27 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:27.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:27 smithi026 ceph-mon[26568]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:27.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:27 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:29.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:29 smithi151 ceph-mon[22663]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:29 smithi026 ceph-mon[21867]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:29.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:29 smithi026 ceph-mon[26568]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:30.252 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:24:31.436 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:31 smithi026 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:31.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:31 smithi026 ceph-mon[26568]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:31.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:31 smithi151 ceph-mon[22663]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:32.111 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:24:32.111 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (12m) 2m ago 20m 24.3M - 0.23.0 ba2b418f427c 1a93078168a0 2024-04-27T02:24:32.111 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 error 2m ago 20m - - 2024-04-27T02:24:32.111 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (2m) 2m ago 19m 52.5M - 3.5 e1d6a67b021e bab26193c34b 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283 running (14m) 2m ago 24m 496M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 210983b6ef19 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (9m) 2m ago 26m 448M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (26m) 2m ago 26m 69.8M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (25m) 2m ago 25m 48.7M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (25m) 2m ago 25m 52.9M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (12m) 2m ago 21m 17.5M - 1.3.1 1dbe0e931976 b2e202622fa1 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (12m) 2m ago 21m 17.1M - 1.3.1 1dbe0e931976 00d8989305ef 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (24m) 2m ago 24m 53.5M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (24m) 2m ago 24m 54.2M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (23m) 2m ago 23m 50.2M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (23m) 2m ago 23m 56.0M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (23m) 2m ago 23m 54.3M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:24:32.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (22m) 2m ago 22m 53.7M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:24:32.113 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (22m) 2m ago 22m 50.3M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:24:32.113 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (21m) 2m ago 21m 54.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:24:32.113 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (2m) 2m ago 21m 51.9M - 2.33.4 514e6a882f6e 0d1b89e74e14 2024-04-27T02:24:32.113 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (19m) 2m ago 19m 75.1M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:24:32.113 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (19m) 2m ago 19m 77.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:24:32.762 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ls' 2024-04-27T02:24:33.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:33 smithi026 ceph-mon[21867]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:33.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:33 smithi026 ceph-mon[21867]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:33.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:33 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:33.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:33 smithi026 ceph-mon[26568]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:33.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:33 smithi026 ceph-mon[26568]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:33.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:33 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:33.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:33 smithi151 ceph-mon[22663]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:33.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:33 smithi151 ceph-mon[22663]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:33.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:33 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:34.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:24:33] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.33.4" 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager ?:9093,9094 1/1 2m ago 21m smithi026=a;count:1 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:grafana ?:3000 0/1 2m ago 21m smithi151=a;count:1 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo ?:5000 1/1 2m ago 19m count:1 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:mgr 2/2 2m ago 25m smithi026=y;smithi151=x;count:2 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:mon 3/3 2m ago 25m smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi151:172.21.15.151=b;count:3 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter ?:9100 2/2 2m ago 21m smithi026=a;smithi151=b;count:2 2024-04-27T02:24:35.001 INFO:teuthology.orchestra.run.smithi026.stdout:osd 8 2m ago - 2024-04-27T02:24:35.002 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus ?:9095 1/1 2m ago 21m smithi151=a;count:1 2024-04-27T02:24:35.002 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo ?:8000 2/2 2m ago 19m count:2 2024-04-27T02:24:35.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:35 smithi026 ceph-mon[21867]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:35.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:35 smithi026 ceph-mon[26568]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:35.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:35 smithi151 ceph-mon[22663]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:35.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-27T02:24:36.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:36 smithi026 ceph-mon[21867]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:36.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:36 smithi026 ceph-mon[26568]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:36.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:36 smithi151 ceph-mon[22663]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:37.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:37 smithi151 ceph-mon[22663]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:37.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:37 smithi026 ceph-mon[21867]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:37.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:37 smithi026 ceph-mon[26568]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:37.727 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:24:37.728 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:24:37.729 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:24:37.729 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:24:37.729 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:24:38.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:38 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/586664233' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:24:38.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:38 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/586664233' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:24:38.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-04-27T02:24:38.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:38 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/586664233' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:24:39.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:39 smithi026 ceph-mon[26568]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:39.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:39 smithi026 ceph-mon[21867]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:39.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:39 smithi151 ceph-mon[22663]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:41.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:41 smithi026 ceph-mon[21867]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:24:41.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:41 smithi026 ceph-mon[21867]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:41.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:41 smithi026 ceph-mon[26568]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:24:41.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:41 smithi026 ceph-mon[26568]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:41.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:41 smithi151 ceph-mon[22663]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:24:41.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:41 smithi151 ceph-mon[22663]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:42.449 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:24:42.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:42 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:42.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:42 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:42.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:42 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:43.093 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-27T02:24:43.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:43.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:24:43.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:43.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:43.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:43.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:43.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:24:43.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:24:43.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 0 op/s 2024-04-27T02:24:43.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:43.695 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:24:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:24:43.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:24:43.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 0 op/s 2024-04-27T02:24:43.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:43 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:43.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:24:43.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:24:43.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:43.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:43.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:43.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:43.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:24:43.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:24:43.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 0 op/s 2024-04-27T02:24:43.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:44.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:24:43] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-04-27T02:24:44.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:44 smithi026 ceph-mon[21867]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:24:44.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:44 smithi026 ceph-mon[26568]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:24:44.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:44 smithi151 ceph-mon[22663]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:24:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:45 smithi026 ceph-mon[21867]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:45.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:45 smithi026 ceph-mon[26568]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:45.708 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:24:45.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:45 smithi151 ceph-mon[22663]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:24:46.392 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (12m) 2m ago 20m 24.3M - 0.23.0 ba2b418f427c 1a93078168a0 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 error 2m ago 20m - - 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (2m) 2m ago 19m 52.5M - 3.5 e1d6a67b021e bab26193c34b 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283 running (14m) 2m ago 25m 496M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 210983b6ef19 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (9m) 2m ago 26m 448M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (26m) 2m ago 26m 69.8M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (25m) 2m ago 25m 48.7M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (25m) 2m ago 25m 52.9M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (12m) 2m ago 21m 17.5M - 1.3.1 1dbe0e931976 b2e202622fa1 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (12m) 2m ago 21m 17.1M - 1.3.1 1dbe0e931976 00d8989305ef 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (24m) 2m ago 24m 53.5M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (24m) 2m ago 24m 54.2M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:24:46.393 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (24m) 2m ago 24m 50.2M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (23m) 2m ago 23m 56.0M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (23m) 2m ago 23m 54.3M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (22m) 2m ago 22m 53.7M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (22m) 2m ago 22m 50.3M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (22m) 2m ago 22m 54.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (3m) 2m ago 21m 51.9M - 2.33.4 514e6a882f6e 0d1b89e74e14 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (19m) 2m ago 19m 75.1M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:24:46.394 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (19m) 2m ago 19m 77.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:24:46.807 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:24:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:24:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:24:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:24:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:24:46.809 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:24:47.083 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:24:47] ENGINE Bus STOPPING 2024-04-27T02:24:47.083 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:24:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:24:47.083 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:24:47] ENGINE Bus STOPPED 2024-04-27T02:24:47.083 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:24:47] ENGINE Bus STARTING 2024-04-27T02:24:47.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='client.25372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:47.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: Failing over to other MGR 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='client.25384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 808 B/s rd, 0 op/s 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: Standby manager daemon x started 2024-04-27T02:24:47.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/58286365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='client.25372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: Failing over to other MGR 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:24:47.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='client.25384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 808 B/s rd, 0 op/s 2024-04-27T02:24:47.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T02:24:47.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/58286365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:24:47.239 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:47.016+0000 7f733d000700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-27T02:24:47.239 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ignoring --setuser ceph since I am not root 2024-04-27T02:24:47.239 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: ignoring --setgroup ceph since I am not root 2024-04-27T02:24:47.239 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mgr[43038]: -- 172.21.15.151:0/1130605707 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5621edd2c5a0 con 0x5621edd00800 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:47.198+0000 7f03c9f5a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='client.25372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: Failing over to other MGR 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='mgr.15270 172.21.15.151:0/3493661668' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:24:47.240 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.241 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='client.25384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:47.241 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 808 B/s rd, 0 op/s 2024-04-27T02:24:47.241 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: Standby manager daemon x started 2024-04-27T02:24:47.241 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/58286365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:24:47.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:24:47] ENGINE Serving on http://:::9283 2024-04-27T02:24:47.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:24:47] ENGINE Bus STARTED 2024-04-27T02:24:47.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:47.389+0000 7f03c9f5a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:24:47.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:47.497+0000 7f03c9f5a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:24:48.014 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:24:48.015 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:24:48.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:47.824+0000 7f03c9f5a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:24:48.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:47 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:47.920+0000 7f03c9f5a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:24:48.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15270 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-27T02:24:48.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: mgrmap e38: y(active, starting, since 1.00268s), standbys: x 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:24:48.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: Manager daemon y is now available 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: [27/Apr/2024:02:24:47] ENGINE Bus STARTING 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: [27/Apr/2024:02:24:47] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: [27/Apr/2024:02:24:47] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:24:48.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:48 smithi151 ceph-mon[22663]: [27/Apr/2024:02:24:47] ENGINE Bus STARTED 2024-04-27T02:24:48.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15270 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-27T02:24:48.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: mgrmap e38: y(active, starting, since 1.00268s), standbys: x 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:24:48.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:24:48.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: Manager daemon y is now available 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: [27/Apr/2024:02:24:47] ENGINE Bus STARTING 2024-04-27T02:24:48.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: [27/Apr/2024:02:24:47] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:24:48.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: [27/Apr/2024:02:24:47] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:24:48.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[21867]: [27/Apr/2024:02:24:47] ENGINE Bus STARTED 2024-04-27T02:24:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15270 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-27T02:24:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: mgrmap e38: y(active, starting, since 1.00268s), standbys: x 2024-04-27T02:24:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:24:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:24:48.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:24:48.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: Manager daemon y is now available 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: [27/Apr/2024:02:24:47] ENGINE Bus STARTING 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: [27/Apr/2024:02:24:47] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: [27/Apr/2024:02:24:47] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:24:48.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:48 smithi026 ceph-mon[26568]: [27/Apr/2024:02:24:47] ENGINE Bus STARTED 2024-04-27T02:24:49.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:49 smithi026 ceph-mon[21867]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-27T02:24:49.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:49 smithi026 ceph-mon[21867]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:49.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:49 smithi026 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:49.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:49 smithi026 ceph-mon[26568]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-27T02:24:49.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:49 smithi026 ceph-mon[26568]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:49.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:49 smithi026 ceph-mon[26568]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:49.191 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:49.193+0000 7f03c9f5a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:24:49.191 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-mon[22663]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-27T02:24:49.191 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-mon[22663]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:24:49.192 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-mon[22663]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:49.532 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:49.368+0000 7f03c9f5a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:24:49.532 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:49.556+0000 7f03c9f5a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[21867]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[21867]: Cluster is now healthy 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[21867]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[26568]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[26568]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[26568]: Cluster is now healthy 2024-04-27T02:24:50.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:50 smithi026 ceph-mon[26568]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-27T02:24:50.152 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:49.883+0000 7f03c9f5a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:24:50.152 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:49.981+0000 7f03c9f5a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:24:50.152 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:50 smithi151 ceph-mon[22663]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:50.153 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:50 smithi151 ceph-mon[22663]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:24:50.153 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:50 smithi151 ceph-mon[22663]: Cluster is now healthy 2024-04-27T02:24:50.153 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:50 smithi151 ceph-mon[22663]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-27T02:24:50.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:50.201+0000 7f03c9f5a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:24:50.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:50.405+0000 7f03c9f5a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:24:51.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:51 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:51.050+0000 7f03c9f5a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:24:51.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:51 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:51.557+0000 7f03c9f5a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:24:51.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:51 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:51.661+0000 7f03c9f5a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:24:52.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:51 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:51.878+0000 7f03c9f5a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:24:52.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-mon[22663]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:52.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-mon[22663]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-27T02:24:52.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[26568]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:52.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[26568]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-27T02:24:52.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:52.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[21867]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-27T02:24:52.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:52 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:52.765 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:52.436+0000 7f03c9f5a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:24:52.765 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:52.537+0000 7f03c9f5a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:24:53.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:52.789+0000 7f03c9f5a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:24:53.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:52 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:52.925+0000 7f03c9f5a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:24:53.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:53.034+0000 7f03c9f5a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:24:53.504 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:53.161+0000 7f03c9f5a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:24:53.504 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:53.260+0000 7f03c9f5a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:24:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-mon[22663]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:53 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:24:53.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[26568]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[21867]: from='client.25130 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:53.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:53.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:53 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:54.171+0000 7f03c9f5a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:54.391+0000 7f03c9f5a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:24:54] ENGINE Bus STARTING 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: CherryPy Checker: 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: The Application mounted at '' has an empty config. 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:24:54] ENGINE Serving on http://:::9283 2024-04-27T02:24:54.474 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[43034]: [27/Apr/2024:02:24:54] ENGINE Bus STARTED 2024-04-27T02:24:54.505 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=warn ts=2024-04-27T02:24:54.243Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:24:54.841 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:24:54.749Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=2 2024-04-27T02:24:54.841 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Standby manager daemon x restarted 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:24:54.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:54.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: Standby manager daemon x started 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:54 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:24:54.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:24:54.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Standby manager daemon x restarted 2024-04-27T02:24:54.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: Standby manager daemon x started 2024-04-27T02:24:54.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:24:54.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:24:54.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:24:54.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:54 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/4154959573' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 114 KiB/s rd, 0 B/s wr, 151 op/s 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[21867]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[21867]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[26568]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 114 KiB/s rd, 0 B/s wr, 151 op/s 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[26568]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:24:55.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:55 smithi026 ceph-mon[26568]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-27T02:24:55.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:55 smithi151 ceph-mon[22663]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 114 KiB/s rd, 0 B/s wr, 151 op/s 2024-04-27T02:24:55.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:55 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:24:55.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:55 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:24:55.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:55 smithi151 ceph-mon[22663]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:24:55.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:55 smithi151 ceph-mon[22663]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-27T02:24:57.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 114 op/s 2024-04-27T02:24:57.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-mon[26568]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 114 op/s 2024-04-27T02:24:57.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:24:57.470+0000 7f8d592a0700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Creating ceph-iscsi config... 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Traceback (most recent call last): 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: return _run_code(code, main_globals, None, 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: exec(code, run_globals) 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:57.605 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Traceback (most recent call last): 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:24:57.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: return future.result(timeout) 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: return self.__get_result() 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: raise self._exception 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: config_blobs=daemon_spec.final_config, 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Creating ceph-iscsi config... 2024-04-27T02:24:57.607 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: Traceback (most recent call last): 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: return _run_code(code, main_globals, None, 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: exec(code, run_globals) 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:57.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:57.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:57.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:57.609 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:24:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:57.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:57 smithi151 ceph-mon[22663]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 114 op/s 2024-04-27T02:24:58.538 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:58 smithi151 ceph-mon[22663]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:58.538 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:24:58.538 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:58.539 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: return future.result(timeout) 2024-04-27T02:24:58.540 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: return self.__get_result() 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: raise self._exception 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: Creating ceph-iscsi config... 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.541 INFO:journalctl@ceph.mon.b.smithi151.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: exec(code, run_globals) 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:58.542 INFO:journalctl@ceph.mon.b.smithi151.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:58.543 INFO:journalctl@ceph.mon.b.smithi151.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.543 INFO:journalctl@ceph.mon.b.smithi151.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.543 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:58 smithi151 ceph-mon[22663]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:24:58.543 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:58 smithi151 ceph-mon[22663]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:24:58.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:58 smithi026 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:58.854 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:24:58.854 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:58.854 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:58.854 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:58.855 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-27T02:24:58.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:58 smithi026 ceph-mon[26568]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:58.858 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:58.859 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2437, in _daemon_action 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 777, in wait_async 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-27T02:24:58.860 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1375, in _create_daemon 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: config_blobs=daemon_spec.final_config, 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1686, in _run_cephadm 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:24:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:24:58.862 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:58 smithi026 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:58 smithi026 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1123, in deploy_daemon 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:24:58.863 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:24:58.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:58 smithi026 ceph-mon[26568]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-27T02:24:58.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:58 smithi026 ceph-mon[26568]: Reconfiguring daemon prometheus.a on smithi151 2024-04-27T02:24:59.689 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:24:59 smithi151 ceph-mon[22663]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 155 op/s 2024-04-27T02:24:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:24:59 smithi026 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 155 op/s 2024-04-27T02:24:59.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:24:59 smithi026 ceph-mon[26568]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 155 op/s 2024-04-27T02:25:00.987 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:00 smithi151 systemd[1]: Stopping Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:00.987 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:00 smithi151 bash[57671]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.011Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-27T02:25:01.296 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-27T02:25:01.297 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[54922]: ts=2024-04-27T02:25:01.012Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-27T02:25:01.297 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 podman[57684]: 2024-04-27 02:25:01.021958568 +0000 UTC m=+0.129478010 container died 0d1b89e74e1423fde785bab62bfca5f03a9f4c92dfcbb779c2b34218b11a15cd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:01.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:01 smithi151 ceph-mon[22663]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:01.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 podman[57684]: 2024-04-27 02:25:01.427839622 +0000 UTC m=+0.535359070 container cleanup 0d1b89e74e1423fde785bab62bfca5f03a9f4c92dfcbb779c2b34218b11a15cd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:01.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 bash[57684]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a 2024-04-27T02:25:01.596 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 bash[57712]: Error: no container with name or ID "ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-27T02:25:01.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:01 smithi026 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:01 smithi026 ceph-mon[26568]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:01.871 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 podman[57699]: 2024-04-27 02:25:01.645879223 +0000 UTC m=+0.626121007 container remove 0d1b89e74e1423fde785bab62bfca5f03a9f4c92dfcbb779c2b34218b11a15cd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:01.871 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-27T02:25:01.871 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 systemd[1]: Stopped Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:01.871 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.311s CPU time. 2024-04-27T02:25:01.872 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 systemd[1]: Starting Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:01.872 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 podman[57782]: 2024-04-27 02:25:01.789097503 +0000 UTC m=+0.018954593 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-27T02:25:01.872 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:01 smithi151 podman[57782]: 2024-04-27 02:25:01.896805693 +0000 UTC m=+0.126662781 container create 70e662819f5d51348b6f5fb8fea7e37657833b7be19f3744f924b6798deeb998 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:02.157 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 podman[57782]: 2024-04-27 02:25:02.107282836 +0000 UTC m=+0.337139924 container init 70e662819f5d51348b6f5fb8fea7e37657833b7be19f3744f924b6798deeb998 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:02.157 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 podman[57782]: 2024-04-27 02:25:02.110060042 +0000 UTC m=+0.339917131 container start 70e662819f5d51348b6f5fb8fea7e37657833b7be19f3744f924b6798deeb998 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:02.157 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.133Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-27T02:25:02.157 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.133Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.133Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.133Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi151 (none))" 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.133Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.133Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.134Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.135Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.136Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.137Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.137Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=10.234µs 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.137Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-27T02:25:02.158 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 bash[57782]: 70e662819f5d51348b6f5fb8fea7e37657833b7be19f3744f924b6798deeb998 2024-04-27T02:25:02.519 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 systemd[1]: Started Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:02.520 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.347Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-04-27T02:25:02.520 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:25:02] ENGINE Bus STOPPING 2024-04-27T02:25:02.520 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:25:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:25:02.520 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:25:02] ENGINE Bus STOPPED 2024-04-27T02:25:02.520 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:25:02] ENGINE Bus STARTING 2024-04-27T02:25:02.520 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:25:02] ENGINE Serving on http://:::9283 2024-04-27T02:25:02.520 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:25:02] ENGINE Bus STARTED 2024-04-27T02:25:02.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:02.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:02.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:25:02.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:02.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.591Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-04-27T02:25:02.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.632Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-04-27T02:25:02.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.668Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-04-27T02:25:02.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.671Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-04-27T02:25:02.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.672Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-04-27T02:25:02.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.672Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=67.538µs wal_replay_duration=534.333013ms total_replay_duration=534.430744ms 2024-04-27T02:25:02.846 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.675Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-27T02:25:02.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.675Z caller=main.go:947 level=info msg="TSDB started" 2024-04-27T02:25:02.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.675Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-27T02:25:02.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.697Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.992052ms db_storage=800ns remote_storage=1.668µs web_handler=365ns query_engine=604ns scrape=561.075µs scrape_sd=94.849µs notify=10.081µs notify_sd=9.187µs rules=21.084892ms 2024-04-27T02:25:02.847 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:02.697Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:02.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:25:02.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:02.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:02 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:03.823 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:03 smithi151 ceph-mon[22663]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:03.823 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:03 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:03.823 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:03 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:25:03.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:03 smithi026 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:03.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:03 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:03.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:03 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:25:03.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:03 smithi026 ceph-mon[26568]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:03.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:03 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:03.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:03 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-27T02:25:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:04 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:25:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:04 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:25:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:04 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:25:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:04 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:04.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:25:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:25:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:25:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:25:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:25:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:25:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:04 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:05.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:05 smithi151 ceph-mon[22663]: Upgrade: Updating mgr.x 2024-04-27T02:25:05.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:05 smithi151 ceph-mon[22663]: Deploying daemon mgr.x on smithi151 2024-04-27T02:25:05.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:05 smithi151 ceph-mon[22663]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:05.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:05 smithi026 ceph-mon[21867]: Upgrade: Updating mgr.x 2024-04-27T02:25:05.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:05 smithi026 ceph-mon[21867]: Deploying daemon mgr.x on smithi151 2024-04-27T02:25:05.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:05 smithi026 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:05.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:05 smithi026 ceph-mon[26568]: Upgrade: Updating mgr.x 2024-04-27T02:25:05.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:05 smithi026 ceph-mon[26568]: Deploying daemon mgr.x on smithi151 2024-04-27T02:25:05.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:05 smithi026 ceph-mon[26568]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 140 op/s 2024-04-27T02:25:05.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:05 smithi151 systemd[1]: Stopping Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:06.234 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:05 smithi151 podman[58123]: 2024-04-27 02:25:05.997591051 +0000 UTC m=+0.128338820 container died 210983b6ef1956da086775b8f817e220629e55583d6818a4c5828ded37c3cbe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-27T02:25:06.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 podman[58123]: 2024-04-27 02:25:06.341251016 +0000 UTC m=+0.471998782 container cleanup 210983b6ef1956da086775b8f817e220629e55583d6818a4c5828ded37c3cbe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-27T02:25:06.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 bash[58123]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x 2024-04-27T02:25:06.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 podman[58137]: 2024-04-27 02:25:06.650597419 +0000 UTC m=+0.656117143 container remove 210983b6ef1956da086775b8f817e220629e55583d6818a4c5828ded37c3cbe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-04-27T02:25:06.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:25:06.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-27T02:25:06.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 systemd[1]: Stopped Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:06.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service: Consumed 58.230s CPU time. 2024-04-27T02:25:06.973 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:06 smithi151 systemd[1]: Starting Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:07.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 podman[58272]: 2024-04-27 02:25:07.116365658 +0000 UTC m=+0.018131115 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:25:07.345 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 podman[58272]: 2024-04-27 02:25:07.218213691 +0000 UTC m=+0.119979164 container create 06ab5f381980ffd80bf51ba80158b6b341667f5bb539ed75e737c8a288acc908 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:25:07.636 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:07 smithi151 ceph-mon[22663]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 53 op/s 2024-04-27T02:25:07.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 podman[58272]: 2024-04-27 02:25:07.503787442 +0000 UTC m=+0.405552872 container init 06ab5f381980ffd80bf51ba80158b6b341667f5bb539ed75e737c8a288acc908 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-27T02:25:07.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 podman[58272]: 2024-04-27 02:25:07.506606975 +0000 UTC m=+0.408372405 container start 06ab5f381980ffd80bf51ba80158b6b341667f5bb539ed75e737c8a288acc908 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb) 2024-04-27T02:25:07.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 ceph-mgr[58291]: -- 172.21.15.151:0/2597407122 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5638b50265a0 con 0x5638b4ffac00 2024-04-27T02:25:07.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 bash[58272]: 06ab5f381980ffd80bf51ba80158b6b341667f5bb539ed75e737c8a288acc908 2024-04-27T02:25:07.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 systemd[1]: Started Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:07.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:07 smithi026 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 53 op/s 2024-04-27T02:25:07.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:07 smithi026 ceph-mon[26568]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 53 op/s 2024-04-27T02:25:07.941 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:07.660+0000 7f5854edc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:25:07.941 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:07.855+0000 7f5854edc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:25:07.941 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:07 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:07.965+0000 7f5854edc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:25:08.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:08.309+0000 7f5854edc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:25:08.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:08 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:08.397+0000 7f5854edc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:25:08.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:08 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:08.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:08 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:08.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:08 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:08.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:08 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:08.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:08 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:08.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:08 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:09.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:08 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:09.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:08 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:09.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:08 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:09.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:09 smithi026 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 53 op/s 2024-04-27T02:25:09.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:09 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2769411672' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:25:09.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:09 smithi026 ceph-mon[26568]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 53 op/s 2024-04-27T02:25:09.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:09 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2769411672' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:25:10.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:09 smithi151 ceph-mon[22663]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 53 op/s 2024-04-27T02:25:10.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:09 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2769411672' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:25:10.023 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:09 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:09.666+0000 7f5854edc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:25:10.023 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:09 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:09.862+0000 7f5854edc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:25:10.023 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:10.047+0000 7f5854edc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:25:10.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:10.351+0000 7f5854edc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:25:10.595 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:10.447+0000 7f5854edc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:25:11.006 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:10.677+0000 7f5854edc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:25:11.006 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:10 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:10.873+0000 7f5854edc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:25:11.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:11 smithi151 ceph-mon[22663]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:25:11.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:11 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:11.518+0000 7f5854edc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:25:12.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:11 smithi026 ceph-mon[21867]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:25:12.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:11 smithi026 ceph-mon[26568]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:25:12.290 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:11 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:11.992+0000 7f5854edc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:25:12.290 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:12.096+0000 7f5854edc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:25:12.290 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:12.313+0000 7f5854edc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:25:12.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:25:12] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-27T02:25:12.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:12 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:12.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:12 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:12.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:12.871+0000 7f5854edc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:25:13.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:12 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:13.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:12 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:13.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:12 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:13.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:12 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:13.216 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:12 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:12.971+0000 7f5854edc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:25:13.467 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:13.240+0000 7f5854edc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:25:13.468 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:13.391+0000 7f5854edc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:25:13.468 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:13.491+0000 7f5854edc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:25:13.808 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:13.612+0000 7f5854edc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:25:13.808 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:13.712+0000 7f5854edc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:25:14.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-mon[22663]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:25:14.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:14.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:13 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:14.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:13 smithi026 ceph-mon[21867]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:25:14.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:13 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:14.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:13 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:14.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:13 smithi026 ceph-mon[26568]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:25:14.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:13 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:14.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:13 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:14.756 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:14.549+0000 7f5854edc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:25:14.756 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:14.780+0000 7f5854edc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-mon[22663]: Standby manager daemon x restarted 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-mon[22663]: Standby manager daemon x started 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-mon[22663]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: [27/Apr/2024:02:25:14] ENGINE Bus STARTING 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: CherryPy Checker: 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: The Application mounted at '' has an empty config. 2024-04-27T02:25:15.095 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:25:15.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: [27/Apr/2024:02:25:14] ENGINE Serving on http://:::9283 2024-04-27T02:25:15.096 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:25:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: [27/Apr/2024:02:25:14] ENGINE Bus STARTED 2024-04-27T02:25:15.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-27T02:25:15.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[21867]: Standby manager daemon x started 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[21867]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[26568]: Standby manager daemon x restarted 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:25:15.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:14 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/1577178039' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:25:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:15 smithi151 ceph-mon[22663]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:25:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:15 smithi151 ceph-mon[22663]: mgrmap e43: y(active, since 28s), standbys: x 2024-04-27T02:25:16.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:15 smithi026 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:25:16.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:15 smithi026 ceph-mon[21867]: mgrmap e43: y(active, since 28s), standbys: x 2024-04-27T02:25:16.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:15 smithi026 ceph-mon[26568]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:25:16.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:15 smithi026 ceph-mon[26568]: mgrmap e43: y(active, since 28s), standbys: x 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 505 B/s rd, 0 op/s 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:25:17.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:25:17.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:25:17.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:17.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:25:17.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:17 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 505 B/s rd, 0 op/s 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:25:17.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 505 B/s rd, 0 op/s 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: Reconfiguring iscsi.foo.smithi026.ofvlhd (dependencies changed)... 2024-04-27T02:25:17.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:25:17.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:25:17.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:17.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: Reconfiguring daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:25:17.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:17 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:18.388 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:25:19.086 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:25:19.086 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (13m) 27s ago 21m 23.5M - 0.23.0 ba2b418f427c 1a93078168a0 2024-04-27T02:25:19.086 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 error 7s ago 20m - - 2024-04-27T02:25:19.086 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (3m) 27s ago 20m 51.3M - 3.5 e1d6a67b021e bab26193c34b 2024-04-27T02:25:19.086 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (11s) 7s ago 25m 64.3M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (10m) 27s ago 27m 477M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (27m) 27s ago 27m 74.2M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (25m) 7s ago 25m 55.2M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (26m) 27s ago 26m 56.0M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (13m) 27s ago 21m 18.1M - 1.3.1 1dbe0e931976 b2e202622fa1 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (12m) 7s ago 21m 17.0M - 1.3.1 1dbe0e931976 00d8989305ef 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (25m) 27s ago 25m 53.9M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (25m) 27s ago 25m 54.6M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (24m) 27s ago 24m 50.4M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (24m) 27s ago 24m 56.4M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (23m) 7s ago 23m 56.9M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (23m) 7s ago 23m 55.2M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (23m) 7s ago 23m 52.5M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:25:19.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (22m) 7s ago 22m 55.9M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:25:19.088 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (17s) 7s ago 22m 53.4M - 2.33.4 514e6a882f6e 70e662819f5d 2024-04-27T02:25:19.088 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (20m) 27s ago 20m 75.7M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:25:19.088 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (20m) 7s ago 20m 78.0M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:25:19.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:19 smithi151 ceph-mon[22663]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:19.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:19 smithi151 ceph-mon[22663]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:19.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:19 smithi151 ceph-mon[22663]: from='client.25468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:19.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:19 smithi026 ceph-mon[21867]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:19.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:19 smithi026 ceph-mon[21867]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:19.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:19 smithi026 ceph-mon[21867]: from='client.25468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:19.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:19 smithi026 ceph-mon[26568]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:19.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:19 smithi026 ceph-mon[26568]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:19.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:19 smithi026 ceph-mon[26568]: from='client.25468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:25:19.507 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:25:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:25:19.878 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:25:19.878 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mgr daemons", 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:25:19.879 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:25:20.343 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:20 smithi026 ceph-mon[21867]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:20.343 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:20 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1019144151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:20.343 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:20 smithi026 ceph-mon[26568]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:20.343 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:20 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1019144151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:20.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:20 smithi151 ceph-mon[22663]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:20.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:20 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1019144151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all mgr 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all crash 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all mds 2024-04-27T02:25:21.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all nfs 2024-04-27T02:25:21.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/1476480051' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all mgr 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-27T02:25:21.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all crash 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all mds 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:25:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all nfs 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:25:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:21.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:21 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/1476480051' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:25:22.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all mgr 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-27T02:25:22.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all crash 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all mds 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all nfs 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:22.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:21 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/1476480051' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:25:22.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:25:22] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-27T02:25:23.354 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:23 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:23.655 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:23 smithi026 podman[75016]: 2024-04-27 02:25:23.430910236 +0000 UTC m=+0.131931027 container died b2e202622fa12e190495997595c9c294d2c295ff789b9264f6587197d4f392f3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:23.955 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:23 smithi026 podman[75016]: 2024-04-27 02:25:23.762133389 +0000 UTC m=+0.463154186 container cleanup b2e202622fa12e190495997595c9c294d2c295ff789b9264f6587197d4f392f3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:23.955 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:23 smithi026 bash[75016]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a 2024-04-27T02:25:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:23 smithi026 ceph-mon[21867]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-27T02:25:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:23 smithi026 ceph-mon[21867]: Upgrade: Updating node-exporter.a (1/2) 2024-04-27T02:25:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:23 smithi026 ceph-mon[21867]: Deploying daemon node-exporter.a on smithi026 2024-04-27T02:25:23.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:23 smithi026 ceph-mon[26568]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-27T02:25:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:23 smithi026 ceph-mon[26568]: Upgrade: Updating node-exporter.a (1/2) 2024-04-27T02:25:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:23 smithi026 ceph-mon[26568]: Deploying daemon node-exporter.a on smithi026 2024-04-27T02:25:24.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:23 smithi151 ceph-mon[22663]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-27T02:25:24.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:23 smithi151 ceph-mon[22663]: Upgrade: Updating node-exporter.a (1/2) 2024-04-27T02:25:24.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:23 smithi151 ceph-mon[22663]: Deploying daemon node-exporter.a on smithi026 2024-04-27T02:25:24.233 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:23 smithi026 podman[75030]: 2024-04-27 02:25:23.979906786 +0000 UTC m=+0.552823953 container remove b2e202622fa12e190495997595c9c294d2c295ff789b9264f6587197d4f392f3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:24.234 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:23 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:25:24.603 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:24 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-27T02:25:24.604 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:24 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:24.604 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:24 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 5.730s CPU time. 2024-04-27T02:25:24.604 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:24 smithi026 systemd[1]: Starting Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:25.060 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:24 smithi026 bash[75171]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-27T02:25:25.354 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:25 smithi026 bash[75171]: Getting image source signatures 2024-04-27T02:25:25.354 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:25 smithi026 bash[75171]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-27T02:25:25.354 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:25 smithi026 bash[75171]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-27T02:25:25.354 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:25 smithi026 bash[75171]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-27T02:25:26.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:25 smithi151 ceph-mon[22663]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:26.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:25 smithi026 ceph-mon[26568]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:26.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:25 smithi026 ceph-mon[21867]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:26.854 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:26 smithi026 bash[75171]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-27T02:25:26.854 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:26 smithi026 bash[75171]: Writing manifest to image destination 2024-04-27T02:25:27.143 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:26 smithi026 podman[75171]: 2024-04-27 02:25:26.817419712 +0000 UTC m=+2.226110159 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-27T02:25:27.143 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:26 smithi026 podman[75171]: 2024-04-27 02:25:26.924086606 +0000 UTC m=+2.332777102 container create a01cf397c18aebf5e6684228738549590ef6a7bbde3a9dcacfaa8f7615e26ce8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:27.604 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 podman[75171]: 2024-04-27 02:25:27.167728523 +0000 UTC m=+2.576418968 container init a01cf397c18aebf5e6684228738549590ef6a7bbde3a9dcacfaa8f7615e26ce8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 podman[75171]: 2024-04-27 02:25:27.171335603 +0000 UTC m=+2.580026047 container start a01cf397c18aebf5e6684228738549590ef6a7bbde3a9dcacfaa8f7615e26ce8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.176Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.176Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=arp 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-27T02:25:27.605 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-27T02:25:27.606 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-27T02:25:27.606 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-27T02:25:27.606 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-27T02:25:27.606 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-27T02:25:27.606 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=edac 2024-04-27T02:25:27.606 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.177Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-27T02:25:27.607 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=os 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=stat 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=time 2024-04-27T02:25:27.608 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=uname 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a[75223]: ts=2024-04-27T02:25:27.178Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 bash[75171]: a01cf397c18aebf5e6684228738549590ef6a7bbde3a9dcacfaa8f7615e26ce8 2024-04-27T02:25:27.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:25:27 smithi026 systemd[1]: Started Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:27.961 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:27 smithi151 ceph-mon[22663]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 1 op/s 2024-04-27T02:25:27.961 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:27 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:27.961 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:27 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:28.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-mon[26568]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 1 op/s 2024-04-27T02:25:28.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:28.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:28.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-mon[21867]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 1 op/s 2024-04-27T02:25:28.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:28.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:27 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:29.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:29 smithi151 ceph-mon[22663]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:29.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:29 smithi026 ceph-mon[21867]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:29.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:29 smithi026 ceph-mon[26568]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:29.845 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:29 smithi151 systemd[1]: Stopping Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:30.205 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:29 smithi151 podman[59641]: 2024-04-27 02:25:29.932527794 +0000 UTC m=+0.122646524 container died 00d8989305ef3b78e897401f2fd8bea63c20cd9baeb5f5ddb6e6ccedb66caf5e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:25:30.205 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:30 smithi151 ceph-mon[22663]: Upgrade: Updating node-exporter.b (2/2) 2024-04-27T02:25:30.205 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:30 smithi151 ceph-mon[22663]: Deploying daemon node-exporter.b on smithi151 2024-04-27T02:25:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:30 smithi026 ceph-mon[21867]: Upgrade: Updating node-exporter.b (2/2) 2024-04-27T02:25:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:30 smithi026 ceph-mon[21867]: Deploying daemon node-exporter.b on smithi151 2024-04-27T02:25:30.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:30 smithi026 ceph-mon[26568]: Upgrade: Updating node-exporter.b (2/2) 2024-04-27T02:25:30.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:30 smithi026 ceph-mon[26568]: Deploying daemon node-exporter.b on smithi151 2024-04-27T02:25:30.530 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 podman[59641]: 2024-04-27 02:25:30.229938268 +0000 UTC m=+0.420057048 container cleanup 00d8989305ef3b78e897401f2fd8bea63c20cd9baeb5f5ddb6e6ccedb66caf5e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:25:30.530 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 bash[59641]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b 2024-04-27T02:25:30.827 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 podman[59656]: 2024-04-27 02:25:30.555866868 +0000 UTC m=+0.626063011 container remove 00d8989305ef3b78e897401f2fd8bea63c20cd9baeb5f5ddb6e6ccedb66caf5e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:25:30.827 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:25:30.827 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-27T02:25:30.827 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 systemd[1]: Stopped Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:30.827 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 5.284s CPU time. 2024-04-27T02:25:31.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:31 smithi151 ceph-mon[22663]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:25:31.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:30 smithi151 systemd[1]: Starting Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:31.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:31 smithi151 bash[59786]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-27T02:25:31.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:31 smithi026 ceph-mon[26568]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:25:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:31 smithi026 ceph-mon[21867]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:25:31.844 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:31 smithi151 bash[59786]: Getting image source signatures 2024-04-27T02:25:31.845 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:31 smithi151 bash[59786]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-27T02:25:31.845 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:31 smithi151 bash[59786]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-27T02:25:31.845 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:31 smithi151 bash[59786]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-27T02:25:32.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:32 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:32.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:32 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:32.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:32 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:32.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:25:32] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:25:33.345 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 bash[59786]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-27T02:25:33.345 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 bash[59786]: Writing manifest to image destination 2024-04-27T02:25:33.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:33.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-mon[22663]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:25:33.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:33 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:33.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:33 smithi026 ceph-mon[21867]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:25:33.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:33 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:33.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:33 smithi026 ceph-mon[26568]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:25:33.826 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 podman[59786]: 2024-04-27 02:25:33.534069057 +0000 UTC m=+2.584879391 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-27T02:25:33.827 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 podman[59786]: 2024-04-27 02:25:33.632773866 +0000 UTC m=+2.683584192 container create 881b4b216b6ba53a2fcf123061cdd3d4ef5db9a1d05f0be7079fbaa2edeb78de (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:25:33.827 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 podman[59786]: 2024-04-27 02:25:33.851999959 +0000 UTC m=+2.902810286 container init 881b4b216b6ba53a2fcf123061cdd3d4ef5db9a1d05f0be7079fbaa2edeb78de (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 podman[59786]: 2024-04-27 02:25:33.854755258 +0000 UTC m=+2.905565592 container start 881b4b216b6ba53a2fcf123061cdd3d4ef5db9a1d05f0be7079fbaa2edeb78de (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.858Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.858Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.858Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.858Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-27T02:25:34.095 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=arp 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-27T02:25:34.096 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=edac 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.859Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-27T02:25:34.097 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=os 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=stat 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-27T02:25:34.098 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=time 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=uname 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b[59836]: ts=2024-04-27T02:25:33.860Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 bash[59786]: 881b4b216b6ba53a2fcf123061cdd3d4ef5db9a1d05f0be7079fbaa2edeb78de 2024-04-27T02:25:34.099 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:25:33 smithi151 systemd[1]: Started Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:35.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:34 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:35.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:34 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:35.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:34 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:35.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:34 smithi151 ceph-mon[22663]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:35.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:35.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:35.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:35.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[26568]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:35.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:35.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:35.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:35.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:34 smithi026 ceph-mon[21867]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:37.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:37 smithi026 ceph-mon[21867]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:37.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:37 smithi026 ceph-mon[26568]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:37.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:37 smithi151 ceph-mon[22663]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:39.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:38 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:38 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:38 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:38 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:39.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:38 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:39 smithi151 ceph-mon[22663]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:40.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:39 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:39 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:39 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:39 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[26568]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:40.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:39 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:42.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:41 smithi151 ceph-mon[22663]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:42.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:41 smithi026 ceph-mon[21867]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:42.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:41 smithi026 ceph-mon[26568]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:42.561 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:25:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:25:42] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-27T02:25:43.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:42 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:43.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:42 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:43.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:42 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:43.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:42 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:43.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:42 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:43.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:42 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:42 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:42 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:42 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.084 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:44.084 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.084 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.084 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:44.084 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:43 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:44.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:44.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:43 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:45.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:44 smithi151 ceph-mon[22663]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:25:45.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:44 smithi151 ceph-mon[22663]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:45.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:44 smithi151 ceph-mon[22663]: Upgrade: Updating prometheus.a 2024-04-27T02:25:45.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:44 smithi026 ceph-mon[26568]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:25:45.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:44 smithi026 ceph-mon[26568]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:45.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:44 smithi026 ceph-mon[26568]: Upgrade: Updating prometheus.a 2024-04-27T02:25:45.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:44 smithi026 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:25:45.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:44 smithi026 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:45.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:44 smithi026 ceph-mon[21867]: Upgrade: Updating prometheus.a 2024-04-27T02:25:46.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:45 smithi151 ceph-mon[22663]: Deploying daemon prometheus.a on smithi151 2024-04-27T02:25:46.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:45 smithi151 ceph-mon[22663]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:46.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:45 smithi026 ceph-mon[21867]: Deploying daemon prometheus.a on smithi151 2024-04-27T02:25:46.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:45 smithi026 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:46.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:45 smithi026 ceph-mon[26568]: Deploying daemon prometheus.a on smithi151 2024-04-27T02:25:46.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:45 smithi026 ceph-mon[26568]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:48.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-mon[22663]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-27T02:25:48.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:48.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:48 smithi026 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-27T02:25:48.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:48 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:48.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:48 smithi026 ceph-mon[26568]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-27T02:25:48.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:48 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:25:48.684 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 systemd[1]: Stopping Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.709Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.710Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.710Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-27T02:25:48.944 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[57796]: ts=2024-04-27T02:25:48.710Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-27T02:25:48.945 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:48 smithi151 podman[61146]: 2024-04-27 02:25:48.719648349 +0000 UTC m=+0.133398582 container died 70e662819f5d51348b6f5fb8fea7e37657833b7be19f3744f924b6798deeb998 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:49.343 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 podman[61146]: 2024-04-27 02:25:49.075708403 +0000 UTC m=+0.489458643 container cleanup 70e662819f5d51348b6f5fb8fea7e37657833b7be19f3744f924b6798deeb998 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:49.343 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 bash[61146]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a 2024-04-27T02:25:49.595 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 podman[61160]: 2024-04-27 02:25:49.368716793 +0000 UTC m=+0.646572783 container remove 70e662819f5d51348b6f5fb8fea7e37657833b7be19f3744f924b6798deeb998 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:49.595 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-27T02:25:49.595 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 systemd[1]: Stopped Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:49.595 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.244s CPU time. 2024-04-27T02:25:50.014 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 systemd[1]: Starting Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:25:50.014 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 podman[61292]: 2024-04-27 02:25:49.811335954 +0000 UTC m=+0.019135557 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-27T02:25:50.014 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:49 smithi151 podman[61292]: 2024-04-27 02:25:49.920309445 +0000 UTC m=+0.128109083 container create 53ac67210d3bfe24663fcd845ed2eec096eb0c61b989e1e0a839a0e16ce930f3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:50.252 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:25:50.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-mon[22663]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 podman[61292]: 2024-04-27 02:25:50.155820951 +0000 UTC m=+0.363620543 container init 53ac67210d3bfe24663fcd845ed2eec096eb0c61b989e1e0a839a0e16ce930f3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 podman[61292]: 2024-04-27 02:25:50.158643732 +0000 UTC m=+0.366443327 container start 53ac67210d3bfe24663fcd845ed2eec096eb0c61b989e1e0a839a0e16ce930f3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.183Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.183Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.183Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi151 (none))" 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.183Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.183Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-27T02:25:50.345 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.185Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.185Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.186Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.186Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.189Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.189Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=11.129µs 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.189Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.195Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.209Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.216Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.219Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.221Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.222Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-04-27T02:25:50.346 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.222Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.222Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.727µs wal_replay_duration=33.692664ms wbl_replay_duration=131ns total_replay_duration=33.753636ms 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.225Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.225Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.225Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 bash[61292]: 53ac67210d3bfe24663fcd845ed2eec096eb0c61b989e1e0a839a0e16ce930f3 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 systemd[1]: Started Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.245Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.962742ms db_storage=749ns remote_storage=1.201µs web_handler=425ns query_engine=638ns scrape=360.825µs scrape_sd=82.631µs notify=10.359µs notify_sd=221.178µs rules=19.041598ms tracing=6.396µs 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.245Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-27T02:25:50.347 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:25:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:25:50.245Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-27T02:25:50.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:50 smithi026 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:50.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:50 smithi026 ceph-mon[26568]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:50.945 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (13m) 12s ago 21m 23.5M - 0.23.0 ba2b418f427c 1a93078168a0 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 error 13s ago 21m - - 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (30s) 12s ago 20m 50.3M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (43s) 13s ago 26m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (10m) 12s ago 27m 500M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (27m) 12s ago 28m 77.8M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (26m) 13s ago 26m 56.3M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (26m) 12s ago 26m 59.2M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (24s) 12s ago 22m 8056k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (17s) 13s ago 22m 5180k - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (25m) 12s ago 25m 54.6M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (25m) 12s ago 25m 55.8M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (25m) 12s ago 25m 50.9M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:25:50.946 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (24m) 12s ago 24m 56.6M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:25:50.947 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (24m) 13s ago 24m 57.0M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:25:50.947 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (24m) 13s ago 24m 55.3M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:25:50.947 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (23m) 13s ago 23m 52.6M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:25:50.947 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (23m) 13s ago 23m 55.8M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:25:50.947 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 starting - - - - 2024-04-27T02:25:50.947 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (21m) 12s ago 21m 76.2M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:25:50.947 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (20m) 13s ago 20m 78.3M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:25:51.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:51 smithi151 ceph-mon[22663]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:51.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:51 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:51.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:51 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:51.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:51 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:51.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:51 smithi151 ceph-mon[22663]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:51.369 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:25:51.369 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:25:51.369 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:25:51.370 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:25:51.371 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:25:51.371 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:25:51.371 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:25:51.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[21867]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[21867]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[26568]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:51.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:51 smithi026 ceph-mon[26568]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:25:51.724 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:25:52.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:52 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:52.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:52 smithi151 ceph-mon[22663]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:52.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:52 smithi151 ceph-mon[22663]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:52.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:52 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/2112825315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:52.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:52 smithi151 ceph-mon[22663]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:52.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:52.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[26568]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:52.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[26568]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:52.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2112825315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:52.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[26568]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:52.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:25:52.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[21867]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:52.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:52.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2112825315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:52.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:52 smithi026 ceph-mon[21867]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:25:54.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:54 smithi151 ceph-mon[22663]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:25:54.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:54.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:54 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:54.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:54 smithi026 ceph-mon[26568]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:25:54.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:54.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:54 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:54 smithi026 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:25:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:54 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.072 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:55 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.072 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:55 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.072 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:55 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.072 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:55 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:56.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:55 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:57.091 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:56 smithi151 ceph-mon[22663]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:57.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:56 smithi026 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:57.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:56 smithi026 ceph-mon[26568]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:58 smithi151 ceph-mon[22663]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:58 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:58 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:58 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:58 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:58 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:59.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[26568]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:25:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:58 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:25:59.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:25:59 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:00.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:25:59 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:01.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:00 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:26:01.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:00 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:26:01.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:00 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-27T02:26:02.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:01 smithi151 ceph-mon[22663]: Upgrade: Updating alertmanager.a 2024-04-27T02:26:02.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:01 smithi151 ceph-mon[22663]: Deploying daemon alertmanager.a on smithi026 2024-04-27T02:26:02.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:01 smithi151 ceph-mon[22663]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:02.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:01 smithi026 ceph-mon[21867]: Upgrade: Updating alertmanager.a 2024-04-27T02:26:02.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:01 smithi026 ceph-mon[21867]: Deploying daemon alertmanager.a on smithi026 2024-04-27T02:26:02.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:01 smithi026 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:02.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:01 smithi026 ceph-mon[26568]: Upgrade: Updating alertmanager.a 2024-04-27T02:26:02.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:01 smithi026 ceph-mon[26568]: Deploying daemon alertmanager.a on smithi026 2024-04-27T02:26:02.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:01 smithi026 ceph-mon[26568]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:02.459 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:26:02] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-04-27T02:26:03.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:02 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:03.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:02 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:03.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:02 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:03.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:02 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:03.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:02 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:03.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:02 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:04.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:03 smithi151 ceph-mon[22663]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-04-27T02:26:04.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:03 smithi026 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-04-27T02:26:04.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:03 smithi026 ceph-mon[26568]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-04-27T02:26:04.104 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:03 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:26:04.481 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:04 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[56235]: level=info ts=2024-04-27T02:26:04.188Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:26:04.481 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:04 smithi026 podman[77275]: 2024-04-27 02:26:04.229525094 +0000 UTC m=+0.148206656 container died 1a93078168a034f1e29fcfdd4acd97e1119ae5e3af9eebcc3d5f3e817ebdcbee (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:26:04.853 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:04 smithi026 podman[77275]: 2024-04-27 02:26:04.605038136 +0000 UTC m=+0.523719697 container cleanup 1a93078168a034f1e29fcfdd4acd97e1119ae5e3af9eebcc3d5f3e817ebdcbee (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:26:04.854 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:04 smithi026 bash[77275]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a 2024-04-27T02:26:05.143 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:04 smithi026 podman[77290]: 2024-04-27 02:26:04.890007493 +0000 UTC m=+0.652856639 container remove 1a93078168a034f1e29fcfdd4acd97e1119ae5e3af9eebcc3d5f3e817ebdcbee (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:26:05.143 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:04 smithi026 podman[77290]: 2024-04-27 02:26:04.973999106 +0000 UTC m=+0.736848207 volume remove 698372093a792565b93992c8cff7a1ffcd48666ef97f5148511c83ba5c3249a8 2024-04-27T02:26:05.578 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:05 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-27T02:26:05.579 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:05 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:26:05.579 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:05 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.622s CPU time. 2024-04-27T02:26:05.579 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:05 smithi026 systemd[1]: Starting Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:26:05.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:05 smithi026 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:05.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:05 smithi026 ceph-mon[26568]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:05.854 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:05 smithi026 podman[77421]: 2024-04-27 02:26:05.506833673 +0000 UTC m=+0.018549350 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-27T02:26:05.854 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:05 smithi026 podman[77421]: 2024-04-27 02:26:05.624909542 +0000 UTC m=+0.136625206 volume create f0ed0d89250ec2855f85c96bfa8c7afc31fd8a377fafc25aa37b58d732b21f98 2024-04-27T02:26:05.854 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:05 smithi026 podman[77421]: 2024-04-27 02:26:05.725459484 +0000 UTC m=+0.237175190 container create 8b1e48c88073747df63e3d5b8490a6f4bc9263066cc5dd83a9f631d3f7214107 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:26:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:05 smithi151 ceph-mon[22663]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:06.354 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 podman[77421]: 2024-04-27 02:26:06.061732068 +0000 UTC m=+0.573447731 container init 8b1e48c88073747df63e3d5b8490a6f4bc9263066cc5dd83a9f631d3f7214107 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:26:06.354 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 podman[77421]: 2024-04-27 02:26:06.064568651 +0000 UTC m=+0.576284323 container start 8b1e48c88073747df63e3d5b8490a6f4bc9263066cc5dd83a9f631d3f7214107 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:26:06.354 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.077Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.077Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.079Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.080Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.104Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.104Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.108Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:06.108Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 bash[77421]: 8b1e48c88073747df63e3d5b8490a6f4bc9263066cc5dd83a9f631d3f7214107 2024-04-27T02:26:06.355 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:06 smithi026 systemd[1]: Started Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:26:07.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:07 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:07.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:07 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:07.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:07 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:07.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:07 smithi151 ceph-mon[22663]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:07.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:07.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:07.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:07.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:07.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:07.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:07.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:07.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:07 smithi026 ceph-mon[26568]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:08.353 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:08.081Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000494761s 2024-04-27T02:26:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:09 smithi151 ceph-mon[22663]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:26:09.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:09 smithi026 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:26:09.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:09 smithi026 ceph-mon[26568]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:26:11.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:11 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:11 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:11 smithi151 ceph-mon[22663]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:11.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:11 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:11 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[26568]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:11.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:11 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:12.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:26:12] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-04-27T02:26:12.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:12 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:12.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:12 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:12.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:12 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:12 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:12 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:12 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:13.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:12 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:13.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:12 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:13.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:12 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:14.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:13 smithi151 ceph-mon[22663]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:14.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:13 smithi026 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:14.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:13 smithi026 ceph-mon[26568]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:15.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:15] ENGINE Bus STOPPING 2024-04-27T02:26:15.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:26:15.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:15] ENGINE Bus STOPPED 2024-04-27T02:26:15.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:15] ENGINE Bus STARTING 2024-04-27T02:26:16.058 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:15] ENGINE Serving on http://:::9283 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:15] ENGINE Bus STARTED 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:16.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:15 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:16.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:15 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:16.353 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:26:16.083Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003134637s 2024-04-27T02:26:16.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:16 smithi151 ceph-mon[22663]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-27T02:26:16.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:16 smithi151 ceph-mon[22663]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:16.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:16 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:26:16.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:16 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-27T02:26:17.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[26568]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-27T02:26:17.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[26568]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:17.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:26:17.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-27T02:26:17.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-27T02:26:17.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:17.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-27T02:26:17.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:16 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-27T02:26:18.043 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:17 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:18.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:17 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:18.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:17 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:18.997 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:18 smithi151 ceph-mon[22663]: Upgrade: Updating grafana.a 2024-04-27T02:26:18.998 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:18 smithi151 ceph-mon[22663]: Deploying daemon grafana.a on smithi151 2024-04-27T02:26:18.998 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:18 smithi151 ceph-mon[22663]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 579 B/s rd, 0 op/s 2024-04-27T02:26:19.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:18 smithi026 ceph-mon[21867]: Upgrade: Updating grafana.a 2024-04-27T02:26:19.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:18 smithi026 ceph-mon[21867]: Deploying daemon grafana.a on smithi151 2024-04-27T02:26:19.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:18 smithi026 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 579 B/s rd, 0 op/s 2024-04-27T02:26:19.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:18 smithi026 ceph-mon[26568]: Upgrade: Updating grafana.a 2024-04-27T02:26:19.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:18 smithi026 ceph-mon[26568]: Deploying daemon grafana.a on smithi151 2024-04-27T02:26:19.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:18 smithi026 ceph-mon[26568]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 579 B/s rd, 0 op/s 2024-04-27T02:26:20.587 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 systemd[1]: Starting Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:26:20.587 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 podman[63266]: 2024-04-27 02:26:20.383583808 +0000 UTC m=+0.018549932 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-04-27T02:26:20.587 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 podman[63266]: 2024-04-27 02:26:20.510429878 +0000 UTC m=+0.145396035 container create 89665f31c1488acfce7a5dd13a3eb9faba4ed0555df3ec0e01811c122b800e86 (image=quay.io/ceph/grafana:9.4.12, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-27T02:26:20.838 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-mon[22663]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:20.838 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 podman[63266]: 2024-04-27 02:26:20.70010082 +0000 UTC m=+0.335066936 container init 89665f31c1488acfce7a5dd13a3eb9faba4ed0555df3ec0e01811c122b800e86 (image=quay.io/ceph/grafana:9.4.12, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-27T02:26:20.838 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 podman[63266]: 2024-04-27 02:26:20.713350164 +0000 UTC m=+0.348316280 container start 89665f31c1488acfce7a5dd13a3eb9faba4ed0555df3ec0e01811c122b800e86 (image=quay.io/ceph/grafana:9.4.12, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-27T02:26:20.838 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 bash[63266]: 89665f31c1488acfce7a5dd13a3eb9faba4ed0555df3ec0e01811c122b800e86 2024-04-27T02:26:20.839 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 systemd[1]: Started Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:26:21.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863296167Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-04-27T02:26:21.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863454278Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863464525Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863467647Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863470163Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863473363Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863475794Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863478183Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863480513Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863483538Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863486003Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863488334Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863490728Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863493073Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-27T02:26:21.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863495402Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863497616Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863517068Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863519627Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=settings t=2024-04-27T02:26:20.863523111Z level=info msg="App mode production" 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=sqlstore t=2024-04-27T02:26:20.863569147Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=sqlstore t=2024-04-27T02:26:20.863580243Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.875083056Z level=info msg="Starting DB migrations" 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.878684619Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.890185734Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.891338388Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.891675398Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.892855112Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-27T02:26:21.097 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.893996605Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.895072094Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.895392672Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.896211156Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.896784079Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.897535905Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.898687752Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.899575715Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.900677412Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-27T02:26:21.098 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.901767247Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-27T02:26:21.099 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.902864279Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-27T02:26:21.099 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.903734084Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-27T02:26:21.099 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.904842908Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-27T02:26:21.099 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.905955929Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-27T02:26:21.099 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.90681379Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.90789807Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.908987186Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.909850518Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.910829324Z level=info msg="Executing migration" id="create alert_image table" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.911553391Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.912907096Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.913261956Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.914117612Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.914958251Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.915070842Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.917830168Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.920636224Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.923413152Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-27T02:26:21.100 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.926243123Z level=info msg="Executing migration" id="create secrets table" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.929115383Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.93536115Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.938109494Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.941078371Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.94735244Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.95366953Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.956456659Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.959324009Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.962135589Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.965110565Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.967829017Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.970745447Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.973566274Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-27T02:26:21.101 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.976306942Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.979238132Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.981912158Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.98482696Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.987512878Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.990162946Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.992803625Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.995576324Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:20.998285555Z level=info msg="Executing migration" id="add correlation config column" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.001063513Z level=info msg="Executing migration" id="create entity_events table" 2024-04-27T02:26:21.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.003800164Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-27T02:26:21.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.006615109Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-27T02:26:21.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.006714888Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-27T02:26:21.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.009400708Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-27T02:26:21.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.009497758Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-27T02:26:21.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.011945Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-27T02:26:21.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.012416726Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-27T02:26:21.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.012820923Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.013687081Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.015655237Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.018393614Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.021120136Z level=info msg="Executing migration" id="Drop public config table" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.023875874Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.024681486Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.025504906Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.02626222Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.027087301Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.030927418Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.032082742Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.033299458Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-27T02:26:21.104 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.03407443Z level=info msg="Executing migration" id="add share column" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.035283382Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.036064085Z level=info msg="Executing migration" id="create file table" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.03687661Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.03766237Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.038548905Z level=info msg="Executing migration" id="create file_meta table" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.0393741Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.040170986Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.041001023Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.042047562Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.04289854Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.04371512Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.044981628Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.045770059Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-27T02:26:21.105 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.046620698Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.047482947Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.048316966Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.049132588Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.050384924Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.051625142Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.052401667Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.065675539Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.066457471Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.069419178Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.073880828Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.076475089Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.079272286Z level=info msg="Executing migration" id="create folder table" 2024-04-27T02:26:21.106 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.082127244Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-27T02:26:21.107 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.085065079Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-27T02:26:21.107 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.087666647Z level=info msg="Executing migration" id="Update folder title length" 2024-04-27T02:26:21.107 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.090415816Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-27T02:26:21.107 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=migrator t=2024-04-27T02:26:21.093034323Z level=info msg="migrations completed" performed=105 skipped=376 duration=214.394323ms 2024-04-27T02:26:21.107 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=sqlstore t=2024-04-27T02:26:21.093355967Z level=info msg="Created default organization" 2024-04-27T02:26:21.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:20 smithi026 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:21.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:20 smithi026 ceph-mon[26568]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:21.595 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=plugin.finder t=2024-04-27T02:26:21.125610519Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-04-27T02:26:21.595 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=secrets t=2024-04-27T02:26:21.12570457Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-27T02:26:21.595 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=query_data t=2024-04-27T02:26:21.127786663Z level=info msg="Query Service initialization" 2024-04-27T02:26:21.595 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=live.push_http t=2024-04-27T02:26:21.130366504Z level=info msg="Live Push Gateway initialization" 2024-04-27T02:26:22.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:22.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:22.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:22.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=infra.usagestats.collector t=2024-04-27T02:26:21.786597809Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-27T02:26:22.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=provisioning.datasources t=2024-04-27T02:26:21.792174884Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-04-27T02:26:22.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=provisioning.datasources t=2024-04-27T02:26:21.792200586Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-04-27T02:26:22.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=provisioning.datasources t=2024-04-27T02:26:21.792359641Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-27T02:26:22.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=provisioning.datasources t=2024-04-27T02:26:21.80399292Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-27T02:26:22.095 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=provisioning.alerting t=2024-04-27T02:26:21.814934889Z level=info msg="starting to provision alerting" 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=provisioning.alerting t=2024-04-27T02:26:21.8149494Z level=info msg="finished to provision alerting" 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=ngalert.state.manager t=2024-04-27T02:26:21.815048795Z level=info msg="Warming state cache for startup" 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=grafanaStorageLogger t=2024-04-27T02:26:21.8151037Z level=info msg="storage starting" 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=ngalert.state.manager t=2024-04-27T02:26:21.815232318Z level=info msg="State cache has been initialized" states=0 duration=183.212µs 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=ticker t=2024-04-27T02:26:21.815329585Z level=info msg=starting first_tick=2024-04-27T02:26:30Z 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=ngalert.multiorg.alertmanager t=2024-04-27T02:26:21.815347406Z level=info msg="starting MultiOrg Alertmanager" 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=http.server t=2024-04-27T02:26:21.817525237Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=sqlstore.transactions t=2024-04-27T02:26:21.83494302Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-27T02:26:22.096 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:26:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=sqlstore.transactions t=2024-04-27T02:26:21.847160505Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-27T02:26:22.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:22.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:21 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:22.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:21 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:22.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:22.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:21 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:22.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:21 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:22.105 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:26:22.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:26:22] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-04-27T02:26:22.776 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:26:22.776 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (17s) 12s ago 22m 15.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:26:22.776 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 starting - - - - 2024-04-27T02:26:22.776 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (62s) 12s ago 21m 50.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:26:22.776 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (75s) 12s ago 26m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (11m) 12s ago 28m 504M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (28m) 12s ago 28m 80.3M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (27m) 12s ago 27m 56.2M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (27m) 12s ago 27m 59.1M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (55s) 12s ago 22m 7948k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (49s) 12s ago 22m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (26m) 12s ago 26m 54.6M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (26m) 12s ago 26m 55.8M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (25m) 12s ago 25m 50.9M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (25m) 12s ago 25m 56.6M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (24m) 12s ago 24m 57.0M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (24m) 12s ago 24m 55.3M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:26:22.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (24m) 12s ago 24m 52.6M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:26:22.778 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (23m) 12s ago 23m 55.8M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:26:22.778 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (32s) 12s ago 23m 47.4M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:26:22.778 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (21m) 12s ago 21m 76.3M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:26:22.778 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (21m) 12s ago 21m 78.4M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:26:23.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:22 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:23.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:22 smithi151 ceph-mon[22663]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:23.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:22 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:23.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:22 smithi026 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:23.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:22 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:23.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:22 smithi026 ceph-mon[26568]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:23.184 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:26:23.185 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:26:23.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:26:23.186 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:26:23.186 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:26:23.186 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:26:23.186 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:26:23.186 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:26:23.528 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:26:23.528 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:26:23.528 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:26:23.528 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-27T02:26:23.529 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-27T02:26:23.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-27T02:26:23.529 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:26:23.529 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-27T02:26:23.529 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading grafana daemons", 2024-04-27T02:26:23.529 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:26:23.529 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:26:23.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[21867]: from='client.25546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.082 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:23 smithi151 ceph-mon[22663]: from='client.25546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.082 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:23 smithi151 ceph-mon[22663]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.082 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:23 smithi151 ceph-mon[22663]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.082 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:23 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3914824242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:24.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[21867]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[21867]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3914824242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[26568]: from='client.25546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[26568]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[26568]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:23 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3914824242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:24.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:24 smithi151 ceph-mon[22663]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:24.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:24 smithi151 ceph-mon[22663]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:25.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:24 smithi026 ceph-mon[26568]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:25.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:24 smithi026 ceph-mon[26568]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:24 smithi026 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:26:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:24 smithi026 ceph-mon[21867]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:26.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:26 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:26.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:26 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:26.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:26 smithi151 ceph-mon[22663]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:26:26.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:26 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:26.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:26 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:26.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:26 smithi026 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:26:26.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:26 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:26.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:26 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:26.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:26 smithi026 ceph-mon[26568]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:26:27.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:27 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:27.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:27 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:27.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:27 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:27.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:27 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:27.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:27 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:27.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:27 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:28.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:28 smithi151 ceph-mon[22663]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:28.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:28 smithi026 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:28.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:28 smithi026 ceph-mon[26568]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:30.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:29] ENGINE Bus STOPPING 2024-04-27T02:26:30.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:26:30.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:29] ENGINE Bus STOPPED 2024-04-27T02:26:30.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:29 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:29] ENGINE Bus STARTING 2024-04-27T02:26:30.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:30] ENGINE Serving on http://:::9283 2024-04-27T02:26:30.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:26:30] ENGINE Bus STARTED 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:26:31.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:26:31.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:26:31.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:26:31.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:30 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:26:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:26:31.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:26:31.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:26:31.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:26:31.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:26:31.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:31.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:26:31.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:26:31.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:26:31.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:26:31.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:26:31.117 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.117 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.117 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.117 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:31.117 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:26:31.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:26:31.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:26:31.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:30 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:32.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[21867]: Upgrade: Finalizing container_image settings 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[21867]: Upgrade: Complete! 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[21867]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[21867]: Cluster is now healthy 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[26568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[26568]: Upgrade: Finalizing container_image settings 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[26568]: Upgrade: Complete! 2024-04-27T02:26:32.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[26568]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:26:32.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:31 smithi026 ceph-mon[26568]: Cluster is now healthy 2024-04-27T02:26:32.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:31 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-27T02:26:32.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:31 smithi151 ceph-mon[22663]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-27T02:26:32.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:31 smithi151 ceph-mon[22663]: Upgrade: Finalizing container_image settings 2024-04-27T02:26:32.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:31 smithi151 ceph-mon[22663]: Upgrade: Complete! 2024-04-27T02:26:32.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:31 smithi151 ceph-mon[22663]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-27T02:26:32.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:31 smithi151 ceph-mon[22663]: Cluster is now healthy 2024-04-27T02:26:32.353 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:26:32] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.43.0" 2024-04-27T02:26:33.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:32 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:33.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:32 smithi151 ceph-mon[22663]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:33.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:32 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:33.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:32 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:33.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:32 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:33.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:33.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[26568]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:33.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:32 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:26:34.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:34 smithi151 ceph-mon[22663]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:34.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:34 smithi026 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:34.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:34 smithi026 ceph-mon[26568]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:37.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:36 smithi151 ceph-mon[22663]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:37.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:36 smithi026 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:37.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:36 smithi026 ceph-mon[26568]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:39.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:38 smithi151 ceph-mon[22663]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:39.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:38 smithi026 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:39.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:38 smithi026 ceph-mon[26568]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:41.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:40 smithi151 ceph-mon[22663]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:41.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:40 smithi026 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:40 smithi026 ceph-mon[26568]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-27T02:26:42.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:26:42] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.43.0" 2024-04-27T02:26:43.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:42 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:43.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:42 smithi151 ceph-mon[22663]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:43.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:42 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:43.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:42 smithi026 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:43.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:42 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:43.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:42 smithi026 ceph-mon[26568]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:44.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:44 smithi151 ceph-mon[22663]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:44.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:44 smithi026 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:44.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:44 smithi026 ceph-mon[26568]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:47.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:46 smithi151 ceph-mon[22663]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:47.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:46 smithi026 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:47.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:46 smithi026 ceph-mon[26568]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:48.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:47 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:48.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:47 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:48.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:47 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:26:49.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:48 smithi151 ceph-mon[22663]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:49.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:48 smithi026 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:49.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:48 smithi026 ceph-mon[26568]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:51.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:50 smithi151 ceph-mon[22663]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:51.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:50 smithi026 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:51.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:50 smithi026 ceph-mon[26568]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:52.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:26:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:26:52] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.43.0" 2024-04-27T02:26:53.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:52 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:53.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:52 smithi151 ceph-mon[22663]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:53.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:52 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:53.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:52 smithi026 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:53.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:52 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:26:53.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:52 smithi026 ceph-mon[26568]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:54.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:54 smithi026 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:54.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:54 smithi026 ceph-mon[26568]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:54.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:54 smithi151 ceph-mon[22663]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:26:54.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-27T02:26:55.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:55 smithi026 ceph-mon[21867]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:55.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:55 smithi026 ceph-mon[26568]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:55.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:55 smithi151 ceph-mon[22663]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:26:56.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:56 smithi026 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:56.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:56 smithi026 ceph-mon[26568]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:56.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:56 smithi151 ceph-mon[22663]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:56.619 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:26:57.223 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:57 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/883230141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:57.223 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:57 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/883230141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:57.249 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-04-27T02:26:57.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:57 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/883230141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:58.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:58 smithi151 ceph-mon[22663]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:58.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:58 smithi026 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:58.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:58 smithi026 ceph-mon[26568]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:26:59.333 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)" 2024-04-27T02:26:59.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:26:59 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3293153556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:59.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:26:59 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3293153556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:26:59.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:26:59 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3293153556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:00.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-04-27T02:27:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:00 smithi026 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:00 smithi026 ceph-mon[26568]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:00.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:00 smithi151 ceph-mon[22663]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:02.028 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:27:02.321 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:27:02] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.43.0" 2024-04-27T02:27:02.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-04-27T02:27:02.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:02.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:02.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3915957517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:02.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:02.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:02.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[26568]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:02.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3915957517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:02.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:02 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:03.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:02 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:03.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:02 smithi151 ceph-mon[22663]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:03.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:02 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3915957517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:03.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:02 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:04.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:04 smithi026 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:04.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:04 smithi026 ceph-mon[26568]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:04.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:04 smithi151 ceph-mon[22663]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:05.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:05 smithi151 ceph-mon[22663]: from='client.25600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:05.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:05 smithi026 ceph-mon[21867]: from='client.25600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:05.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:05 smithi026 ceph-mon[26568]: from='client.25600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:06.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:06 smithi151 ceph-mon[22663]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:06.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:06 smithi026 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:06.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:06 smithi026 ceph-mon[26568]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:06.725 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:27:07.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:27:09.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:08 smithi026 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:09.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:08 smithi026 ceph-mon[26568]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:08 smithi151 ceph-mon[22663]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:09.337 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:27:09.337 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:27:09.337 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:27:09.337 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:27:09.337 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:27:09.338 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:27:09.338 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:27:09.338 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:27:09.338 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:27:10.035 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:27:10.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:10 smithi026 ceph-mon[21867]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:10.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:10 smithi026 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:10.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:10 smithi026 ceph-mon[26568]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:10.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:10 smithi026 ceph-mon[26568]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:11.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:10 smithi151 ceph-mon[22663]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:11.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:10 smithi151 ceph-mon[22663]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:12.090 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:27:12.353 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:27:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:27:12] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.43.0" 2024-04-27T02:27:12.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-04-27T02:27:13.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:12 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:13.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:12 smithi026 ceph-mon[26568]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:13.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:12 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/502931572' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:27:13.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:12 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:13.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:12 smithi026 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:13.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:12 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/502931572' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:27:13.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:12 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:13.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:12 smithi151 ceph-mon[22663]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:13.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:12 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/502931572' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:27:14.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:14 smithi026 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:14.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:14 smithi026 ceph-mon[26568]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:14.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:14 smithi151 ceph-mon[22663]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:15.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:15 smithi151 ceph-mon[22663]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:15.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:15 smithi026 ceph-mon[21867]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:15.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:15 smithi026 ceph-mon[26568]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:16.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:16 smithi151 ceph-mon[22663]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mon", "hosts": "smithi151", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:16.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:16 smithi151 ceph-mon[22663]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:16.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:16 smithi026 ceph-mon[21867]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mon", "hosts": "smithi151", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:16.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:16 smithi026 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:16.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:16 smithi026 ceph-mon[26568]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mon", "hosts": "smithi151", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:16.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:16 smithi026 ceph-mon[26568]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:17 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:17.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:17 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:17.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:17 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:18.332 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:27:18.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:18 smithi151 ceph-mon[22663]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:18.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:18 smithi026 ceph-mon[26568]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:18.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:18 smithi026 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:19.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-27T02:27:19.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[21867]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:27:19.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:19.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:19.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:19 smithi151 ceph-mon[22663]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:27:19.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:19 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:19.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:19 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:19.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:19 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:19.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:19 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:19.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:19 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:19.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:19.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:19.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:19.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[26568]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:27:19.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:19.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:19.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:19.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:19.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:19 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:20.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:20 smithi026 ceph-mon[21867]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:27:20.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:20 smithi026 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:20.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:20 smithi026 ceph-mon[26568]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:27:20.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:20 smithi026 ceph-mon[26568]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:20.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:20 smithi151 ceph-mon[22663]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:27:20.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:20 smithi151 ceph-mon[22663]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:21.077 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:27:21.761 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (76s) 71s ago 23m 15.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (61s) 56s ago 22m 49.2M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (2m) 71s ago 22m 50.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (2m) 56s ago 27m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (12m) 71s ago 29m 504M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (29m) 71s ago 29m 80.3M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (28m) 56s ago 28m 56.2M 2048M 17.2.0 e1d6a67b021e 8dcb4692da87 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (28m) 71s ago 28m 59.1M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:27:21.762 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (114s) 71s ago 23m 7948k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:27:21.763 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (108s) 56s ago 23m 15.3M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:27:21.763 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (27m) 71s ago 27m 54.6M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:27:21.765 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (27m) 71s ago 27m 55.8M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:27:21.765 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (26m) 71s ago 26m 50.9M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (26m) 71s ago 26m 56.6M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (25m) 56s ago 25m 57.0M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (25m) 56s ago 25m 55.3M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (25m) 56s ago 25m 52.6M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (24m) 56s ago 24m 55.8M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (91s) 56s ago 24m 47.9M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (22m) 71s ago 22m 76.3M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:27:21.766 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (22m) 56s ago 22m 78.4M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:27:22.176 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:27:22.176 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:27:22.176 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-27T02:27:22.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:27:22.176 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:27:22.176 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:27:22.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:27:22.177 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:27:22.304 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:27:22] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-04-27T02:27:22.529 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:27:22.529 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:27:22.529 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:27:22.529 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi151", 2024-04-27T02:27:22.529 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:27:22.529 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/1 daemons upgraded", 2024-04-27T02:27:22.529 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-04-27T02:27:22.530 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:27:22.530 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:27:23.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:23.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='client.25630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='client.? 172.21.15.26:0/3863142676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:27:23.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:22 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-27T02:27:23.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:23.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='client.25630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/3863142676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='client.25630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/3863142676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:27:23.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:22 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-27T02:27:24.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:23 smithi151 ceph-mon[22663]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:27:24.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:23 smithi151 ceph-mon[22663]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:27:24.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:23 smithi151 ceph-mon[22663]: Upgrade: Setting container_image for all mgr 2024-04-27T02:27:24.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:23 smithi151 ceph-mon[22663]: Upgrade: It appears safe to stop mon.b 2024-04-27T02:27:24.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:23 smithi151 ceph-mon[22663]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:24.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[26568]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[26568]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all mgr 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[26568]: Upgrade: It appears safe to stop mon.b 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[26568]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[21867]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[21867]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all mgr 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[21867]: Upgrade: It appears safe to stop mon.b 2024-04-27T02:27:24.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:23 smithi026 ceph-mon[21867]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:24.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:24.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[26568]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:27:24.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:24 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:25.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:24 smithi151 ceph-mon[22663]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:25.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:24 smithi151 ceph-mon[22663]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:25.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:24 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:27:25.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:24 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:27:25.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:24 smithi151 ceph-mon[22663]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:26.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:25 smithi151 systemd[1]: Stopping Ceph mon.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:27:26.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:25 smithi151 ceph-mon[22663]: Upgrade: Updating mon.b 2024-04-27T02:27:26.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:25 smithi151 ceph-mon[22663]: Deploying daemon mon.b on smithi151 2024-04-27T02:27:26.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b[22658]: 2024-04-27T02:27:25.982+0000 7f3eb7b1d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:27:26.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b[22658]: 2024-04-27T02:27:25.982+0000 7f3eb7b1d700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-27T02:27:26.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:25 smithi026 ceph-mon[21867]: Upgrade: Updating mon.b 2024-04-27T02:27:26.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:25 smithi026 ceph-mon[21867]: Deploying daemon mon.b on smithi151 2024-04-27T02:27:26.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:25 smithi026 ceph-mon[26568]: Upgrade: Updating mon.b 2024-04-27T02:27:26.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:25 smithi026 ceph-mon[26568]: Deploying daemon mon.b on smithi151 2024-04-27T02:27:26.434 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:26 smithi151 podman[64725]: 2024-04-27 02:27:26.218424329 +0000 UTC m=+0.324452445 container died 8dcb4692da8749083ac35d6c8d7f2b01df146c28b6e9d6155e5051b72d5f4a70 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_CLEAN=True, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, name=centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870) 2024-04-27T02:27:26.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:26 smithi151 podman[64725]: 2024-04-27 02:27:26.541768671 +0000 UTC m=+0.647796782 container cleanup 8dcb4692da8749083ac35d6c8d7f2b01df146c28b6e9d6155e5051b72d5f4a70 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.expose-services=, RELEASE=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, architecture=x86_64, io.openshift.tags=base centos centos-stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870) 2024-04-27T02:27:26.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:26 smithi151 bash[64725]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b 2024-04-27T02:27:27.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:26 smithi151 podman[64739]: 2024-04-27 02:27:26.726649182 +0000 UTC m=+0.510189330 container remove 8dcb4692da8749083ac35d6c8d7f2b01df146c28b6e9d6155e5051b72d5f4a70 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , RELEASE=HEAD, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, release=754, com.redhat.component=centos-stream-container, vcs-type=git, vendor=Red Hat, Inc.) 2024-04-27T02:27:27.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:26 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-27T02:27:27.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:26 smithi151 systemd[1]: Stopped Ceph mon.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:27:27.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:26 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.b.service: Consumed 17.990s CPU time. 2024-04-27T02:27:27.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 systemd[1]: Starting Ceph mon.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:27:27.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 podman[64870]: 2024-04-27 02:27:27.344287385 +0000 UTC m=+0.115282350 container create cb8177cb84042c8fac73fc0fa7ed1b7e779f783f79ee475fc4059441df9511ca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-27T02:27:27.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 podman[64870]: 2024-04-27 02:27:27.2477982 +0000 UTC m=+0.018793188 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:27:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 podman[64870]: 2024-04-27 02:27:27.621911916 +0000 UTC m=+0.392906878 container init cb8177cb84042c8fac73fc0fa7ed1b7e779f783f79ee475fc4059441df9511ca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-27T02:27:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 podman[64870]: 2024-04-27 02:27:27.625763463 +0000 UTC m=+0.396758425 container start cb8177cb84042c8fac73fc0fa7ed1b7e779f783f79ee475fc4059441df9511ca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-27T02:27:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: set uid:gid to 167:167 (ceph:ceph) 2024-04-27T02:27:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-27T02:27:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: pidfile_write: ignore empty --pid-file 2024-04-27T02:27:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: load: jerasure load: lrc 2024-04-27T02:27:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: RocksDB version: 7.9.2 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Git sha 0 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: DB SUMMARY 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: DB Session ID: P6QFM1SHUQAHVCFCMWI5 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: CURRENT file: CURRENT 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: IDENTITY file: IDENTITY 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2836 Bytes 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 3200939 ; 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.error_if_exists: 0 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.create_if_missing: 0 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.paranoid_checks: 1 2024-04-27T02:27:27.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-27T02:27:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-27T02:27:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-27T02:27:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.env: 0x556dda139c60 2024-04-27T02:27:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.fs: PosixFileSystem 2024-04-27T02:27:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.info_log: 0x556ddbee7aa0 2024-04-27T02:27:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_file_opening_threads: 16 2024-04-27T02:27:27.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.statistics: (nil) 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.use_fsync: 0 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_log_file_size: 0 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.keep_log_file_num: 1000 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.recycle_log_file_num: 0 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.allow_fallocate: 1 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.allow_mmap_reads: 0 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.allow_mmap_writes: 0 2024-04-27T02:27:27.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.use_direct_reads: 0 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.create_missing_column_families: 0 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.db_log_dir: 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.wal_dir: 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.advise_random_on_open: 1 2024-04-27T02:27:27.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.db_write_buffer_size: 0 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.write_buffer_manager: 0x556ddbeba820 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.rate_limiter: (nil) 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.wal_recovery_mode: 2 2024-04-27T02:27:27.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.enable_thread_tracking: 0 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.enable_pipelined_write: 0 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.unordered_write: 0 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.row_cache: None 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.wal_filter: None 2024-04-27T02:27:27.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.allow_ingest_behind: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.two_write_queues: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.manual_wal_flush: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.wal_compression: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.atomic_flush: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.log_readahead_size: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.best_efforts_recovery: 0 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-27T02:27:27.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.allow_data_in_errors: 0 2024-04-27T02:27:27.853 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.db_host_id: __hostname__ 2024-04-27T02:27:27.853 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-27T02:27:27.853 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_background_jobs: 2 2024-04-27T02:27:27.853 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_background_compactions: -1 2024-04-27T02:27:27.853 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_subcompactions: 1 2024-04-27T02:27:27.853 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_total_wal_size: 0 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_open_files: -1 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bytes_per_sync: 0 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-27T02:27:27.854 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_readahead_size: 0 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_background_flushes: -1 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Compression algorithms supported: 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kZSTD supported: 0 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kXpressCompression supported: 0 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kLZ4HCCompression supported: 1 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kZlibCompression supported: 1 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kSnappyCompression supported: 1 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kLZ4Compression supported: 1 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: kBZip2Compression supported: 0 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-27T02:27:27.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.merge_operator: 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_filter: None 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_filter_factory: None 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.sst_partitioner_factory: None 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556ddbee7ba0) 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks: 1 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_top_level_index_and_filter: 1 2024-04-27T02:27:27.856 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_type: 0 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_index_type: 0 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_shortening: 1 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: checksum: 4 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: no_block_cache: 0 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache: 0x556ddbeeb090 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_name: BinnedLRUCache 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_options: 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: capacity : 536870912 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: num_shard_bits : 4 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: strict_capacity_limit : 0 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: high_pri_pool_ratio: 0.000 2024-04-27T02:27:27.857 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_compressed: (nil) 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: persistent_cache: (nil) 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size: 4096 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size_deviation: 10 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_restart_interval: 16 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_block_restart_interval: 1 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: metadata_block_size: 4096 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: partition_filters: 0 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: use_delta_encoding: 1 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: filter_policy: bloomfilter 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: whole_key_filtering: 1 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: verify_compression: 0 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: read_amp_bytes_per_bit: 0 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: format_version: 5 2024-04-27T02:27:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_index_compression: 1 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_align: 0 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_auto_readahead_size: 262144 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout: prepopulate_block_cache: 0 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout: initial_auto_readahead_size: 8192 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout: num_file_reads_for_auto_readahead: 2 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.write_buffer_size: 33554432 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_write_buffer_number: 2 2024-04-27T02:27:27.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression: NoCompression 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression: Disabled 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.prefix_extractor: nullptr 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.num_levels: 7 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-27T02:27:27.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.level: 32767 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.strategy: 0 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-27T02:27:27.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.enabled: false 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.target_file_size_base: 67108864 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-27T02:27:27.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.arena_block_size: 1048576 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.disable_auto_compactions: 0 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-27T02:27:27.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.table_properties_collectors: 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.inplace_update_support: 0 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-27T02:27:27.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.bloom_locality: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.max_successive_merges: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.paranoid_file_checks: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.force_consistency_checks: 1 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.report_bg_io_stats: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.ttl: 2592000 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.enable_blob_files: false 2024-04-27T02:27:27.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.min_blob_size: 0 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.blob_file_size: 268435456 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.blob_file_starting_level: 0 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-27T02:27:27.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28900, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d4f06554-cb1b-4170-8189-728f801b1f7e 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184847667193, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184847675426, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 1949535, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28905, "largest_seqno": 30803, "table_properties": {"data_size": 1942465, "index_size": 4198, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1861, "raw_key_size": 17441, "raw_average_key_size": 24, "raw_value_size": 1927329, "raw_average_value_size": 2662, "num_data_blocks": 190, "num_entries": 724, "num_filter_entries": 724, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714184847, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d4f06554-cb1b-4170-8189-728f801b1f7e", "db_session_id": "P6QFM1SHUQAHVCFCMWI5", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184847675548, "job": 1, "event": "recovery_finished"} 2024-04-27T02:27:27.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556ddbfdc000 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: DB pointer 0x556ddbfc6000 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** DB Stats ** 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T02:27:27.869 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: L0 1/0 1.86 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 533.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: L6 1/0 8.95 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: Sum 2/0 10.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 533.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 533.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-27T02:27:27.870 INFO:journalctl@ceph.mon.b.smithi151.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-27T02:27:27.871 INFO:journalctl@ceph.mon.b.smithi151.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T02:27:27.871 INFO:journalctl@ceph.mon.b.smithi151.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 533.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:27:27.871 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T02:27:27.871 INFO:journalctl@ceph.mon.b.smithi151.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-27T02:27:27.871 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T02:27:27.871 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T02:27:27.872 INFO:journalctl@ceph.mon.b.smithi151.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-27T02:27:27.872 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-27T02:27:27.872 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-27T02:27:27.872 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-27T02:27:27.872 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-27T02:27:27.872 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative compaction: 0.00 GB write, 85.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval compaction: 0.00 GB write, 85.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout: Block cache BinnedLRUCache@0x556ddbeeb090#2 capacity: 512.00 MB usage: 6.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: starting mon.b rank 2 at public addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] at bind addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???) e3 preinit fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???).mds e1 new map 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???).mds e1 print_map 2024-04-27T02:27:27.873 INFO:journalctl@ceph.mon.b.smithi151.stdout: e1 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout: legacy client fscid: -1 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout: No filesystems configured 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???).osd e110 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 bash[64870]: cb8177cb84042c8fac73fc0fa7ed1b7e779f783f79ee475fc4059441df9511ca 2024-04-27T02:27:27.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 systemd[1]: Started Ceph mon.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:27:28.274 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:27 smithi151 ceph-mon[64897]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:28.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:27 smithi026 ceph-mon[26568]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:28.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:27 smithi026 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: mon.b calling monitor election 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: mon.c calling monitor election 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: mon.a calling monitor election 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: fsmap 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: mgrmap e43: y(active, since 2m), standbys: x 2024-04-27T02:27:28.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: overall HEALTH_OK 2024-04-27T02:27:28.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:28.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:28.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:28 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:29.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: mon.b calling monitor election 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: mon.c calling monitor election 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: mon.a calling monitor election 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: fsmap 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: mgrmap e43: y(active, since 2m), standbys: x 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: overall HEALTH_OK 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:29.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: mon.b calling monitor election 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: mon.c calling monitor election 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: mon.a calling monitor election 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: fsmap 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: mgrmap e43: y(active, since 2m), standbys: x 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: overall HEALTH_OK 2024-04-27T02:27:29.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:29.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:29.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:28 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:30.153 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:29 smithi151 ceph-mon[64897]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:29 smithi026 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:30.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:29 smithi026 ceph-mon[26568]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:32.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:27:32] "GET /metrics HTTP/1.1" 200 37934 "" "Prometheus/2.43.0" 2024-04-27T02:27:33.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:32 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:33.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:32 smithi151 ceph-mon[64897]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:33.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:32 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:33.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:32 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:33.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:32 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:33.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:33.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[26568]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:33.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:33.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:33.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-27T02:27:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:33.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:33.104 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:27:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:27:32.684+0000 7f8d5c2a6700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-27T02:27:34.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:34 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:34.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:34 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:34.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:34 smithi151 ceph-mon[64897]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:34.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:34 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:34.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:34 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:34.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:34 smithi026 ceph-mon[26568]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:34.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:34 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:34.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:34 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:34.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:34 smithi026 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:37.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:27:37.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:27:37.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:27:37.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:27:37.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:27:37.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:27:37.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:36 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:37.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:27:37.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:27:37.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:27:37.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:36 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:27:38.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Detected new or changed devices on smithi151 2024-04-27T02:27:38.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all crash 2024-04-27T02:27:38.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mds 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nfs 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all node-exporter 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all prometheus 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all alertmanager 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all grafana 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all loki 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all promtail 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Finalizing container_image settings 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:27:38.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:27:38.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: Upgrade: Complete! 2024-04-27T02:27:38.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:37 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Detected new or changed devices on smithi151 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all crash 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all mds 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all nfs 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all node-exporter 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all prometheus 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all alertmanager 2024-04-27T02:27:38.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all grafana 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all loki 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all promtail 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Finalizing container_image settings 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:27:38.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi151 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all crash 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all mds 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:27:38.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all nfs 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all node-exporter 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all prometheus 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all alertmanager 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all grafana 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all loki 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all promtail 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Finalizing container_image settings 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:27:38.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:27:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: Upgrade: Complete! 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: Upgrade: Complete! 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:38.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:38.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:38.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:27:38.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:27:38.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:27:38.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:38.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:37 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:27:39.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:38 smithi151 ceph-mon[64897]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:39.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:38 smithi026 ceph-mon[26568]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:39.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:38 smithi026 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:41.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:40 smithi151 ceph-mon[64897]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:41.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:40 smithi026 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:40 smithi026 ceph-mon[26568]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:42.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:27:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:27:42] "GET /metrics HTTP/1.1" 200 37934 "" "Prometheus/2.43.0" 2024-04-27T02:27:43.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:42 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:43.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:42 smithi151 ceph-mon[64897]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:43.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:42 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:43.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:42 smithi026 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:43.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:42 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:43.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:42 smithi026 ceph-mon[26568]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:44.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:44 smithi151 ceph-mon[64897]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:44.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:44 smithi026 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:44.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:44 smithi026 ceph-mon[26568]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:47.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:46 smithi151 ceph-mon[64897]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:47.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:46 smithi026 ceph-mon[26568]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:47.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:46 smithi026 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:48.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:47 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:48.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:47 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:48.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:47 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:27:49.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:48 smithi151 ceph-mon[64897]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:49.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:48 smithi026 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:49.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:48 smithi026 ceph-mon[26568]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:51.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:50 smithi151 ceph-mon[64897]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:51.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:50 smithi026 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:51.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:50 smithi026 ceph-mon[26568]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:52.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:27:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:27:52] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-04-27T02:27:52.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:52 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:52.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:52 smithi026 ceph-mon[21867]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:52.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:52 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:52.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:52 smithi026 ceph-mon[26568]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:53.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:52 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:27:53.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:52 smithi151 ceph-mon[64897]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:53.496 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:27:54.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:54 smithi151 ceph-mon[64897]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:54.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:54 smithi151 ceph-mon[64897]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:54.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:54 smithi026 ceph-mon[21867]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:54.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:54 smithi026 ceph-mon[21867]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:54.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:54 smithi026 ceph-mon[26568]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:54.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:54 smithi026 ceph-mon[26568]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:27:55.419 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:27:55.419 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (109s) 104s ago 23m 15.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:27:55.419 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (95s) 23s ago 23m 79.4M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:27:55.419 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (2m) 104s ago 22m 50.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:27:55.419 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (2m) 23s ago 28m 453M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:27:55.419 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (12m) 104s ago 30m 504M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (30m) 104s ago 30m 80.3M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (28s) 23s ago 28m 19.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (28m) 104s ago 28m 59.1M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 104s ago 24m 7948k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (2m) 23s ago 24m 15.6M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (28m) 104s ago 28m 54.6M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (27m) 104s ago 27m 55.8M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (27m) 104s ago 27m 50.9M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (26m) 104s ago 26m 56.6M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (26m) 23s ago 26m 57.0M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (26m) 23s ago 26m 55.5M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (25m) 23s ago 25m 52.6M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (25m) 23s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (2m) 23s ago 24m 60.0M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (23m) 104s ago 23m 76.3M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:27:55.420 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (23m) 23s ago 23m 78.8M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:27:56.020 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-04-27T02:27:57.048 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:56 smithi151 ceph-mon[64897]: from='client.25666 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:57.048 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:56 smithi151 ceph-mon[64897]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:57.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:56 smithi026 ceph-mon[21867]: from='client.25666 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:57.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:56 smithi026 ceph-mon[21867]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:57.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:56 smithi026 ceph-mon[26568]: from='client.25666 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:27:57.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:56 smithi026 ceph-mon[26568]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:58.083 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:27:58.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:27:58.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:58 smithi026 ceph-mon[21867]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:58.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:27:58 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2960670886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:58.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:58 smithi026 ceph-mon[26568]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:58.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:27:58 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2960670886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:27:59.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:58 smithi151 ceph-mon[64897]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:27:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:27:58 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2960670886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:00.706 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:28:00.706 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:28:00.706 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:28:00.706 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:28:00.706 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:28:00.706 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:28:00.706 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:28:00.707 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:28:00.707 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:28:00.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:00 smithi026 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:00.979 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:00 smithi026 ceph-mon[26568]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:01.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:00 smithi151 ceph-mon[64897]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:01.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:28:01.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:01 smithi026 ceph-mon[21867]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:01.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:01 smithi026 ceph-mon[26568]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:02.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:01 smithi151 ceph-mon[64897]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:02.582 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:28:02] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-04-27T02:28:02.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:02 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:02.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:02 smithi026 ceph-mon[21867]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:02.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:02 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:02.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:02 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:02.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:02 smithi026 ceph-mon[26568]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:02.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:02 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:03.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:02 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:03.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:02 smithi151 ceph-mon[64897]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:03.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:02 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:03.419 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:28:03.991 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-04-27T02:28:04.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:04 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/2590609695' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:28:04.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:04 smithi026 ceph-mon[21867]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:04.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:04 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/2590609695' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:28:04.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:04 smithi026 ceph-mon[26568]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:04.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:04 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2590609695' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:28:04.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:04 smithi151 ceph-mon[64897]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:07.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:06 smithi151 ceph-mon[64897]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:07.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:06 smithi026 ceph-mon[26568]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:06 smithi026 ceph-mon[21867]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:07.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:07 smithi026 ceph-mon[21867]: from='client.15783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:07.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:07 smithi026 ceph-mon[21867]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mon", "hosts": "smithi026", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:07.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:07 smithi026 ceph-mon[26568]: from='client.15783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:07.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:07 smithi026 ceph-mon[26568]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mon", "hosts": "smithi026", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:08.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:07 smithi151 ceph-mon[64897]: from='client.15783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:08.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:07 smithi151 ceph-mon[64897]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "mon", "hosts": "smithi026", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:08.365 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:28:09.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[26568]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:09.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:09.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:09.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:09.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:09.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:09.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[21867]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:09.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:09.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:09.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:09.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:09.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:08 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:08 smithi151 ceph-mon[64897]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:08 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:08 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:08 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:08 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:09.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:08 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:09.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-27T02:28:10.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:09 smithi151 ceph-mon[64897]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:28:10.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:09 smithi151 ceph-mon[64897]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:28:10.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:09 smithi026 ceph-mon[21867]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:28:10.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:09 smithi026 ceph-mon[21867]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:28:10.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:09 smithi026 ceph-mon[26568]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:28:10.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:09 smithi026 ceph-mon[26568]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:28:11.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:10 smithi151 ceph-mon[64897]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:11.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:10 smithi026 ceph-mon[21867]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:11.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:10 smithi026 ceph-mon[26568]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: Upgrade: Setting container_image for all mgr 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:12.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: Upgrade: It appears safe to stop mon.a 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[26568]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:28:12.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:12.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:12.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: Upgrade: Setting container_image for all mgr 2024-04-27T02:28:12.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:12.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:28:12.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-27T02:28:12.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: Upgrade: It appears safe to stop mon.a 2024-04-27T02:28:12.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-mon[21867]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:12.481 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:28:12] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-04-27T02:28:12.488 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:28:12.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mgr 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: Upgrade: It appears safe to stop mon.a 2024-04-27T02:28:12.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:12 smithi151 ceph-mon[64897]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:13.167 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:28:13.167 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 2m ago 24m 15.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:28:13.167 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (112s) 41s ago 23m 79.4M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:28:13.167 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (2m) 2m ago 23m 50.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:28:13.167 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (3m) 41s ago 28m 453M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:28:13.167 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (13m) 2m ago 30m 504M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:28:13.167 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (30m) 2m ago 30m 80.3M 2048M 17.2.0 e1d6a67b021e f821e7b12391 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (45s) 41s ago 28m 19.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (29m) 2m ago 29m 59.1M 2048M 17.2.0 e1d6a67b021e bb095f0bbd82 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 2m ago 24m 7948k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (2m) 41s ago 24m 15.6M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (28m) 2m ago 28m 54.6M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (27m) 2m ago 27m 55.8M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (27m) 2m ago 27m 50.9M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (27m) 2m ago 27m 56.6M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (26m) 41s ago 26m 57.0M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (26m) 41s ago 26m 55.5M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (26m) 41s ago 26m 52.6M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (25m) 41s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (2m) 41s ago 24m 60.0M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (23m) 2m ago 23m 76.3M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:28:13.168 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (23m) 41s ago 23m 78.8M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:28:13.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:13 smithi026 ceph-mon[21867]: from='client.15795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:13.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:13 smithi026 ceph-mon[21867]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:13.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:13 smithi026 ceph-mon[26568]: from='client.15795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:13.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:13 smithi026 ceph-mon[26568]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-27T02:28:13.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:28:13.582 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:28:13.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:13.582 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:28:13.582 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-27T02:28:13.582 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:28:13.582 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:28:13.582 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:28:13.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:13 smithi151 ceph-mon[64897]: from='client.15795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:13.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:13 smithi151 ceph-mon[64897]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi026", 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/2 daemons upgraded", 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:28:13.938 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:28:14.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: from='client.25714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:14.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: Upgrade: Updating mon.a 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: Deploying daemon mon.a on smithi026 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: from='client.? 172.21.15.26:0/748330593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[26568]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:14.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: from='client.25714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: Upgrade: Updating mon.a 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: Deploying daemon mon.a on smithi026 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: from='client.? 172.21.15.26:0/748330593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:14.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:14 smithi026 ceph-mon[21867]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:14.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: from='client.25714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: Upgrade: Updating mon.a 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='mgr.y' 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: Deploying daemon mon.a on smithi026 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/748330593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:14.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:14 smithi151 ceph-mon[64897]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:15.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:15 smithi026 systemd[1]: Stopping Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:28:15.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:15 smithi026 ceph-mon[21867]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:15.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:15 smithi026 ceph-mon[26568]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:15.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a[21863]: 2024-04-27T02:28:15.439+0000 7f780b0ff700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:28:15.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a[21863]: 2024-04-27T02:28:15.439+0000 7f780b0ff700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-27T02:28:15.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:15 smithi026 podman[82098]: 2024-04-27 02:28:15.552551098 +0000 UTC m=+0.218102138 container died f821e7b12391c592a84a0d3aa3af4a9dbf5233ad2ab0b0b4ac7ad15468949875 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, ceph=True, io.openshift.tags=base centos centos-stream, name=centos-stream, io.openshift.expose-services=, version=8) 2024-04-27T02:28:15.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:15 smithi151 ceph-mon[64897]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:16.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:15 smithi026 podman[82098]: 2024-04-27 02:28:15.940605159 +0000 UTC m=+0.606156199 container cleanup f821e7b12391c592a84a0d3aa3af4a9dbf5233ad2ab0b0b4ac7ad15468949875 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, ceph=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.buildah.version=1.19.8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container) 2024-04-27T02:28:16.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:15 smithi026 bash[82098]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a 2024-04-27T02:28:16.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:16 smithi026 podman[82112]: 2024-04-27 02:28:16.175472471 +0000 UTC m=+0.624162350 container remove f821e7b12391c592a84a0d3aa3af4a9dbf5233ad2ab0b0b4ac7ad15468949875 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, release=754, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, version=8) 2024-04-27T02:28:16.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:16 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-27T02:28:16.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:16 smithi026 systemd[1]: Stopped Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:28:16.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:16 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service: Consumed 28.671s CPU time. 2024-04-27T02:28:16.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:16 smithi026 systemd[1]: Starting Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:28:16.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:16 smithi026 podman[82241]: 2024-04-27 02:28:16.647690282 +0000 UTC m=+0.018754504 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:28:16.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:16 smithi026 podman[82241]: 2024-04-27 02:28:16.776433897 +0000 UTC m=+0.147498094 container create 576ac55bff7b36784cc0bbc28fa2f198d97cf2a7fc93b1128b64c15f82f2151c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, ceph=True) 2024-04-27T02:28:17.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 podman[82241]: 2024-04-27 02:28:17.045940935 +0000 UTC m=+0.417005140 container init 576ac55bff7b36784cc0bbc28fa2f198d97cf2a7fc93b1128b64c15f82f2151c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 podman[82241]: 2024-04-27 02:28:17.049052176 +0000 UTC m=+0.420116378 container start 576ac55bff7b36784cc0bbc28fa2f198d97cf2a7fc93b1128b64c15f82f2151c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: set uid:gid to 167:167 (ceph:ceph) 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: pidfile_write: ignore empty --pid-file 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: load: jerasure load: lrc 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: RocksDB version: 7.9.2 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Git sha 0 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: DB SUMMARY 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: DB Session ID: UGR24X0BLC2AYE60W5ZT 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: CURRENT file: CURRENT 2024-04-27T02:28:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: IDENTITY file: IDENTITY 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3102 Bytes 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 394025 ; 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.error_if_exists: 0 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.create_if_missing: 0 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.paranoid_checks: 1 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.env: 0x55a17ae84c60 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.fs: PosixFileSystem 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.info_log: 0x55a17d1e7aa0 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_file_opening_threads: 16 2024-04-27T02:28:17.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.statistics: (nil) 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.use_fsync: 0 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_log_file_size: 0 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.keep_log_file_num: 1000 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.recycle_log_file_num: 0 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.allow_fallocate: 1 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.allow_mmap_reads: 0 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.allow_mmap_writes: 0 2024-04-27T02:28:17.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.use_direct_reads: 0 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.create_missing_column_families: 0 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.db_log_dir: 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.wal_dir: 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.advise_random_on_open: 1 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.db_write_buffer_size: 0 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.write_buffer_manager: 0x55a17d1ba820 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-27T02:28:17.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.rate_limiter: (nil) 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.wal_recovery_mode: 2 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.enable_thread_tracking: 0 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.enable_pipelined_write: 0 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.unordered_write: 0 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.row_cache: None 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.wal_filter: None 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-27T02:28:17.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.allow_ingest_behind: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.two_write_queues: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.manual_wal_flush: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.wal_compression: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.atomic_flush: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.log_readahead_size: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.best_efforts_recovery: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.allow_data_in_errors: 0 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.db_host_id: __hostname__ 2024-04-27T02:28:17.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_background_jobs: 2 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_background_compactions: -1 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_subcompactions: 1 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_total_wal_size: 0 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_open_files: -1 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bytes_per_sync: 0 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-27T02:28:17.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_readahead_size: 0 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_background_flushes: -1 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Compression algorithms supported: 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kZSTD supported: 0 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kXpressCompression supported: 0 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kLZ4HCCompression supported: 1 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kZlibCompression supported: 1 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kSnappyCompression supported: 1 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kLZ4Compression supported: 1 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: kBZip2Compression supported: 0 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-27T02:28:17.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.merge_operator: 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_filter: None 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_filter_factory: None 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.sst_partitioner_factory: None 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a17d1e7ba0) 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2024-04-27T02:28:17.363 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 4 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x55a17d1eb090 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2024-04-27T02:28:17.364 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: bloomfilter 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 5 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_auto_readahead_size: 262144 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: prepopulate_block_cache: 0 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: initial_auto_readahead_size: 8192 2024-04-27T02:28:17.365 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.write_buffer_size: 33554432 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_write_buffer_number: 2 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression: NoCompression 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression: Disabled 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.prefix_extractor: nullptr 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.num_levels: 7 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-27T02:28:17.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.level: 32767 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.strategy: 0 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.enabled: false 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-27T02:28:17.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.target_file_size_base: 67108864 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-27T02:28:17.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.arena_block_size: 1048576 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.disable_auto_compactions: 0 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-27T02:28:17.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.table_properties_collectors: 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.inplace_update_support: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.bloom_locality: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.max_successive_merges: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.paranoid_file_checks: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.force_consistency_checks: 1 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.report_bg_io_stats: 0 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.ttl: 2592000 2024-04-27T02:28:17.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.enable_blob_files: false 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.min_blob_size: 0 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.blob_file_size: 268435456 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.blob_file_starting_level: 0 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-27T02:28:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29319, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 41458d63-454e-4133-a9cc-5993f3632732 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184897079095, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184897081326, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 385699, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29320, "largest_seqno": 29740, "table_properties": {"data_size": 383284, "index_size": 953, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 4105, "raw_average_key_size": 23, "raw_value_size": 379551, "raw_average_value_size": 2144, "num_data_blocks": 44, "num_entries": 177, "num_filter_entries": 177, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714184897, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "41458d63-454e-4133-a9cc-5993f3632732", "db_session_id": "UGR24X0BLC2AYE60W5ZT", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184897081420, "job": 1, "event": "recovery_finished"} 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a17d2dc000 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: DB pointer 0x55a17d2c6000 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-27T02:28:17.372 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 1/0 376.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 262.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: L6 1/0 9.67 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 10.04 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 262.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-27T02:28:17.373 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 262.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 262.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-27T02:28:17.374 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-27T02:28:17.375 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-27T02:28:17.375 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-27T02:28:17.375 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 82.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T02:28:17.375 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 82.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T02:28:17.375 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-27T02:28:17.376 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache BinnedLRUCache@0x55a17d1eb090#2 capacity: 512.00 MB usage: 1.58 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-27T02:28:17.376 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2024-04-27T02:28:17.376 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T02:28:17.376 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-27T02:28:17.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:17.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???) e3 preinit fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???).mds e1 new map 2024-04-27T02:28:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???).mds e1 print_map 2024-04-27T02:28:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout: e1 2024-04-27T02:28:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-27T02:28:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-27T02:28:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: -1 2024-04-27T02:28:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-27T02:28:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout: No filesystems configured 2024-04-27T02:28:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???).osd e110 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-27T02:28:17.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:28:17.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:28:17.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:28:17.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 ceph-mon[82260]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-04-27T02:28:17.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 bash[82241]: 576ac55bff7b36784cc0bbc28fa2f198d97cf2a7fc93b1128b64c15f82f2151c 2024-04-27T02:28:17.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:17 smithi026 systemd[1]: Started Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:28:18.844 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: ignoring --setuser ceph since I am not root 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: ignoring --setgroup ceph since I am not root 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mgr[58291]: -- 172.21.15.151:0/3952752761 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5563a638c5a0 con 0x5563a6361000 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:18.695+0000 7fbb43a36200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: mon.a calling monitor election 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: monmap epoch 3 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:18.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: last_changed 2024-04-27T01:59:21.330741+0000 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: created 2024-04-27T01:57:44.453030+0000 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: min_mon_release 17 (quincy) 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: election_strategy: 1 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: fsmap 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: overall HEALTH_OK 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: from='mgr.15360 ' entity='' 2024-04-27T02:28:18.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-mon[64897]: mgrmap e44: y(active, since 3m), standbys: x 2024-04-27T02:28:18.854 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ignoring --setuser ceph since I am not root 2024-04-27T02:28:18.854 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ignoring --setgroup ceph since I am not root 2024-04-27T02:28:18.854 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mgr[60490]: -- 172.21.15.26:0/3828597506 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5560b16425a0 con 0x5560b1616800 2024-04-27T02:28:18.854 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:18.699+0000 7fc811375200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-27T02:28:18.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:18.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: mon.a calling monitor election 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: monmap epoch 3 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: last_changed 2024-04-27T01:59:21.330741+0000 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: created 2024-04-27T01:57:44.453030+0000 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: min_mon_release 17 (quincy) 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: election_strategy: 1 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: fsmap 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:28:18.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: overall HEALTH_OK 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: from='mgr.15360 ' entity='' 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[82260]: mgrmap e44: y(active, since 3m), standbys: x 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: from='mgr.15360 172.21.15.26:0/2272748520' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: mon.a calling monitor election 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: monmap epoch 3 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: last_changed 2024-04-27T01:59:21.330741+0000 2024-04-27T02:28:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: created 2024-04-27T01:57:44.453030+0000 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: min_mon_release 17 (quincy) 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: election_strategy: 1 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: fsmap 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: osdmap e110: 8 total, 8 up, 8 in 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: overall HEALTH_OK 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: from='mgr.15360 ' entity='' 2024-04-27T02:28:18.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-mon[26568]: mgrmap e44: y(active, since 3m), standbys: x 2024-04-27T02:28:19.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:18.880+0000 7fbb43a36200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:28:19.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:18.989+0000 7fbb43a36200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:28:19.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:19.299+0000 7fbb43a36200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:28:19.294 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:18.890+0000 7fc811375200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:28:19.294 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:18.997+0000 7fc811375200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-27T02:28:19.594 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:19.385+0000 7fbb43a36200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:28:19.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:19.317+0000 7fc811375200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-27T02:28:19.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:19.406+0000 7fc811375200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-27T02:28:20.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:20 smithi026 ceph-mon[82260]: mgrmap e45: y(active, since 3m), standbys: x 2024-04-27T02:28:20.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:20 smithi026 ceph-mon[26568]: mgrmap e45: y(active, since 3m), standbys: x 2024-04-27T02:28:20.816 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:20 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:20.660+0000 7fc811375200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:28:20.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:20 smithi151 ceph-mon[64897]: mgrmap e45: y(active, since 3m), standbys: x 2024-04-27T02:28:20.844 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:20.581+0000 7fbb43a36200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-27T02:28:20.844 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:20.750+0000 7fbb43a36200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:28:21.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:20 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:20.839+0000 7fc811375200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-27T02:28:21.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:21.017+0000 7fc811375200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:28:21.190 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:20.924+0000 7fbb43a36200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-27T02:28:21.191 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:21.215+0000 7fbb43a36200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:28:21.490 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:21.308+0000 7fbb43a36200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:28:21.490 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:21.514+0000 7fbb43a36200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:28:21.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:21.320+0000 7fc811375200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-27T02:28:21.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:21.417+0000 7fc811375200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-27T02:28:21.844 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:21.712+0000 7fbb43a36200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:28:22.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:21.633+0000 7fc811375200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-27T02:28:22.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:21.835+0000 7fc811375200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-27T02:28:22.594 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:22.327+0000 7fbb43a36200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:28:22.853 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:22.478+0000 7fc811375200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-27T02:28:23.035 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:22.757+0000 7fbb43a36200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:28:23.035 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:22.855+0000 7fbb43a36200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:28:23.035 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:23.059+0000 7fbb43a36200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:28:23.238 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:22.940+0000 7fc811375200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-27T02:28:23.238 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:23.044+0000 7fc811375200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-27T02:28:23.238 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:23.262+0000 7fc811375200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-27T02:28:23.796 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:28:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:28:23.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:28:23.796 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:28:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:28:23.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:28:23.844 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:23.593+0000 7fbb43a36200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:28:23.844 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:23.687+0000 7fbb43a36200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:28:24.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:23.821+0000 7fc811375200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-27T02:28:24.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:23.920+0000 7fc811375200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-27T02:28:24.261 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:23.935+0000 7fbb43a36200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:28:24.261 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:24.071+0000 7fbb43a36200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:28:24.261 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:24.169+0000 7fbb43a36200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:28:24.262 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:24.285+0000 7fbb43a36200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:28:24.525 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:24.179+0000 7fc811375200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-27T02:28:24.525 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:24.322+0000 7fc811375200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-27T02:28:24.525 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:24.426+0000 7fc811375200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-27T02:28:24.525 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:24.550+0000 7fc811375200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-27T02:28:24.594 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:24.383+0000 7fbb43a36200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:28:24.853 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:24.651+0000 7fc811375200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-27T02:28:25.480 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:25.215+0000 7fbb43a36200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:28:25.480 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:25.442+0000 7fbb43a36200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:28:25.480 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: [27/Apr/2024:02:28:25] ENGINE Bus STARTING 2024-04-27T02:28:25.809 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:28:25.588Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2024-04-27T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[26568]: Standby manager daemon x restarted 2024-04-27T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[26568]: Standby manager daemon x started 2024-04-27T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[26568]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[82260]: Standby manager daemon x restarted 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[82260]: Standby manager daemon x started 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[82260]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[82260]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[82260]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-mon[82260]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:28:25.811 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:25.514+0000 7fc811375200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-27T02:28:25.811 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:25.747+0000 7fc811375200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-27T02:28:25.811 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:28:25] ENGINE Bus STARTING 2024-04-27T02:28:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-mon[64897]: Standby manager daemon x restarted 2024-04-27T02:28:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-mon[64897]: Standby manager daemon x started 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-mon[64897]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-mon[64897]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-mon[64897]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-mon[64897]: from='mgr.? 172.21.15.151:0/835707192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: CherryPy Checker: 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: The Application mounted at '' has an empty config. 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: [27/Apr/2024:02:28:25] ENGINE Serving on http://:::9283 2024-04-27T02:28:25.845 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:28:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x[58286]: [27/Apr/2024:02:28:25] ENGINE Bus STARTED 2024-04-27T02:28:26.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: CherryPy Checker: 2024-04-27T02:28:26.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: The Application mounted at '' has an empty config. 2024-04-27T02:28:26.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:26.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:28:25] ENGINE Serving on http://:::9283 2024-04-27T02:28:26.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:25 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:28:25] ENGINE Bus STARTED 2024-04-27T02:28:26.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: mgrmap e46: y(active, since 3m), standbys: x 2024-04-27T02:28:26.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: Active manager daemon y restarted 2024-04-27T02:28:26.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: Activating manager daemon y 2024-04-27T02:28:26.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:26.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: mgrmap e47: y(active, starting, since 0.00719803s), standbys: x 2024-04-27T02:28:26.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:28:26.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: Manager daemon y is now available 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:26.761+0000 7fc7ce81b700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-27T02:28:26.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: mgrmap e46: y(active, since 3m), standbys: x 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: Active manager daemon y restarted 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: Activating manager daemon y 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: mgrmap e47: y(active, starting, since 0.00719803s), standbys: x 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:28:26.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: Manager daemon y is now available 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:28:26.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:26 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: mgrmap e46: y(active, since 3m), standbys: x 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: Active manager daemon y restarted 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: Activating manager daemon y 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: mgrmap e47: y(active, starting, since 0.00719803s), standbys: x 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-27T02:28:26.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: Manager daemon y is now available 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:26.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:26.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-27T02:28:26.847 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-27T02:28:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[82260]: [27/Apr/2024:02:28:26] ENGINE Bus STARTING 2024-04-27T02:28:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[82260]: [27/Apr/2024:02:28:26] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:28:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[82260]: [27/Apr/2024:02:28:26] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:28:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[82260]: [27/Apr/2024:02:28:26] ENGINE Bus STARTED 2024-04-27T02:28:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[82260]: mgrmap e48: y(active, since 1.00895s), standbys: x 2024-04-27T02:28:27.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[26568]: [27/Apr/2024:02:28:26] ENGINE Bus STARTING 2024-04-27T02:28:27.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[26568]: [27/Apr/2024:02:28:26] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:28:27.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[26568]: [27/Apr/2024:02:28:26] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:28:27.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[26568]: [27/Apr/2024:02:28:26] ENGINE Bus STARTED 2024-04-27T02:28:27.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:27 smithi026 ceph-mon[26568]: mgrmap e48: y(active, since 1.00895s), standbys: x 2024-04-27T02:28:27.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:27 smithi151 ceph-mon[64897]: [27/Apr/2024:02:28:26] ENGINE Bus STARTING 2024-04-27T02:28:27.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:27 smithi151 ceph-mon[64897]: [27/Apr/2024:02:28:26] ENGINE Serving on http://172.21.15.26:8765 2024-04-27T02:28:27.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:27 smithi151 ceph-mon[64897]: [27/Apr/2024:02:28:26] ENGINE Serving on https://172.21.15.26:7150 2024-04-27T02:28:27.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:27 smithi151 ceph-mon[64897]: [27/Apr/2024:02:28:26] ENGINE Bus STARTED 2024-04-27T02:28:27.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:27 smithi151 ceph-mon[64897]: mgrmap e48: y(active, since 1.00895s), standbys: x 2024-04-27T02:28:28.353 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:28:28 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:28:28.096Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=5 2024-04-27T02:28:28.722 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:28 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:28.723 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:28 smithi151 ceph-mon[64897]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:28.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:28 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:28.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:28 smithi026 ceph-mon[82260]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:28.764 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:28 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:28.764 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:28 smithi026 ceph-mon[26568]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:29.574 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:29 smithi151 ceph-mon[64897]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:29.574 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:29 smithi151 ceph-mon[64897]: mgrmap e49: y(active, since 2s), standbys: x 2024-04-27T02:28:29.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:29 smithi026 ceph-mon[82260]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:29.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:29 smithi026 ceph-mon[82260]: mgrmap e49: y(active, since 2s), standbys: x 2024-04-27T02:28:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:29 smithi026 ceph-mon[26568]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:29 smithi026 ceph-mon[26568]: mgrmap e49: y(active, since 2s), standbys: x 2024-04-27T02:28:31.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:31 smithi151 ceph-mon[64897]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:31.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:31 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:31 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:31 smithi151 ceph-mon[64897]: mgrmap e50: y(active, since 4s), standbys: x 2024-04-27T02:28:31.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:31 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:31 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[82260]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[82260]: mgrmap e50: y(active, since 4s), standbys: x 2024-04-27T02:28:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[26568]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:28:31.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[26568]: mgrmap e50: y(active, since 4s), standbys: x 2024-04-27T02:28:31.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:31.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:31 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.458 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:28:32] "GET /metrics HTTP/1.1" 200 35201 "" "Prometheus/2.43.0" 2024-04-27T02:28:32.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.783 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:32 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:32 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:32 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:32 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:32.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:32 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:33.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:33 smithi151 ceph-mon[64897]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-27T02:28:33.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:33 smithi026 ceph-mon[82260]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-27T02:28:33.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:33 smithi026 ceph-mon[26568]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-27T02:28:35.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:35 smithi151 ceph-mon[64897]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:28:35.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:35 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:35 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:35 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:35 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:35 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:35.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:35 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:35.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[26568]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:28:35.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:35.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[82260]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-27T02:28:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:35.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:35 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:36.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: Detected new or changed devices on smithi026 2024-04-27T02:28:36.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:28:36.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:36.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:28:36.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:36 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-27T02:28:36.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: Detected new or changed devices on smithi026 2024-04-27T02:28:36.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:28:36.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:28:36.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:28:36.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: Detected new or changed devices on smithi026 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:36.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:36.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-27T02:28:36.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:36 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-27T02:28:37.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[82260]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:28:37.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[82260]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-27T02:28:37.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[82260]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:28:37.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[82260]: Upgrade: It appears safe to stop mon.c 2024-04-27T02:28:37.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[26568]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:28:37.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[26568]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-27T02:28:37.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[26568]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:28:37.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:37 smithi026 ceph-mon[26568]: Upgrade: It appears safe to stop mon.c 2024-04-27T02:28:37.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:37 smithi151 ceph-mon[64897]: Updating smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:28:37.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:37 smithi151 ceph-mon[64897]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-27T02:28:37.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:37 smithi151 ceph-mon[64897]: Updating smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-27T02:28:37.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:37 smithi151 ceph-mon[64897]: Upgrade: It appears safe to stop mon.c 2024-04-27T02:28:38.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:38.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:38.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:38.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:38.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:38.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[26568]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:38.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:38.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:38.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:38.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:38 smithi026 ceph-mon[26568]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:38.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:38 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:38.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:38 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:38.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:38 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:38.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:38 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:38.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:38 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:39.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-mon[82260]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:28:39.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-mon[82260]: Upgrade: Updating mon.c 2024-04-27T02:28:39.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-mon[82260]: Deploying daemon mon.c on smithi026 2024-04-27T02:28:39.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-mon[26568]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:28:39.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-mon[26568]: Upgrade: Updating mon.c 2024-04-27T02:28:39.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-mon[26568]: Deploying daemon mon.c on smithi026 2024-04-27T02:28:39.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:39 smithi151 ceph-mon[64897]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-27T02:28:39.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:39 smithi151 ceph-mon[64897]: Upgrade: Updating mon.c 2024-04-27T02:28:39.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:39 smithi151 ceph-mon[64897]: Deploying daemon mon.c on smithi026 2024-04-27T02:28:39.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:39 smithi026 systemd[1]: Stopping Ceph mon.c for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:28:39.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c[26564]: 2024-04-27T02:28:39.860+0000 7ff5e2f94700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:28:39.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:39 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c[26564]: 2024-04-27T02:28:39.860+0000 7ff5e2f94700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-27T02:28:40.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:39 smithi026 podman[87051]: 2024-04-27 02:28:39.95287567 +0000 UTC m=+0.190179771 container died bb095f0bbd82d5fadf008d650e436e7444313863395a9a0bb137b66fd8373db6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, RELEASE=HEAD, ceph=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, distribution-scope=public, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-type=git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-27T02:28:40.444 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:40 smithi026 podman[87051]: 2024-04-27 02:28:40.29410702 +0000 UTC m=+0.531411113 container cleanup bb095f0bbd82d5fadf008d650e436e7444313863395a9a0bb137b66fd8373db6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, vendor=Red Hat, Inc., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, version=8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.k8s.display-name=CentOS Stream 8, release=754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-27T02:28:40.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:40 smithi026 bash[87051]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c 2024-04-27T02:28:40.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:40 smithi026 podman[87065]: 2024-04-27 02:28:40.470130914 +0000 UTC m=+0.511054288 container remove bb095f0bbd82d5fadf008d650e436e7444313863395a9a0bb137b66fd8373db6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, release=754, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-type=git, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-27T02:28:40.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:40 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-27T02:28:40.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:40 smithi026 systemd[1]: Stopped Ceph mon.c for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:28:40.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:40 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.c.service: Consumed 17.671s CPU time. 2024-04-27T02:28:41.220 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:40 smithi026 systemd[1]: Starting Ceph mon.c for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:28:41.220 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 podman[87195]: 2024-04-27 02:28:41.11325278 +0000 UTC m=+0.103976235 container create 0757865e99087bf49c382685996c8705e2d4384080bfe5fc75b834781aaa247b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-27T02:28:41.220 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 podman[87195]: 2024-04-27 02:28:41.027526945 +0000 UTC m=+0.018250382 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 podman[87195]: 2024-04-27 02:28:41.357938066 +0000 UTC m=+0.348661479 container init 0757865e99087bf49c382685996c8705e2d4384080bfe5fc75b834781aaa247b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 podman[87195]: 2024-04-27 02:28:41.36127224 +0000 UTC m=+0.351995659 container start 0757865e99087bf49c382685996c8705e2d4384080bfe5fc75b834781aaa247b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: set uid:gid to 167:167 (ceph:ceph) 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: pidfile_write: ignore empty --pid-file 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: load: jerasure load: lrc 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: RocksDB version: 7.9.2 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Git sha 0 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: DB SUMMARY 2024-04-27T02:28:41.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: DB Session ID: JSPGKQGNFG1WJBIIK8DV 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: CURRENT file: CURRENT 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: IDENTITY file: IDENTITY 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3217 Bytes 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2677707 ; 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.error_if_exists: 0 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.create_if_missing: 0 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.paranoid_checks: 1 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.env: 0x561f671a7c60 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.fs: PosixFileSystem 2024-04-27T02:28:41.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.info_log: 0x561f69175aa0 2024-04-27T02:28:41.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_file_opening_threads: 16 2024-04-27T02:28:41.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.statistics: (nil) 2024-04-27T02:28:41.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.use_fsync: 0 2024-04-27T02:28:41.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_log_file_size: 0 2024-04-27T02:28:41.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-27T02:28:41.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-27T02:28:41.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.keep_log_file_num: 1000 2024-04-27T02:28:41.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.recycle_log_file_num: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.allow_fallocate: 1 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.allow_mmap_reads: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.allow_mmap_writes: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.use_direct_reads: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.create_missing_column_families: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.db_log_dir: 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.wal_dir: 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-27T02:28:41.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.advise_random_on_open: 1 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.db_write_buffer_size: 0 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.write_buffer_manager: 0x561f69148820 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.rate_limiter: (nil) 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.wal_recovery_mode: 2 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.enable_thread_tracking: 0 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.enable_pipelined_write: 0 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.unordered_write: 0 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-27T02:28:41.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.row_cache: None 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.wal_filter: None 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.allow_ingest_behind: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.two_write_queues: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.manual_wal_flush: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.wal_compression: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.atomic_flush: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.log_readahead_size: 0 2024-04-27T02:28:41.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.best_efforts_recovery: 0 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.allow_data_in_errors: 0 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.db_host_id: __hostname__ 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_background_jobs: 2 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_background_compactions: -1 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_subcompactions: 1 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_total_wal_size: 0 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-27T02:28:41.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_open_files: -1 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bytes_per_sync: 0 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_readahead_size: 0 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_background_flushes: -1 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Compression algorithms supported: 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kZSTD supported: 0 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kXpressCompression supported: 0 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kLZ4HCCompression supported: 1 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kZlibCompression supported: 1 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kSnappyCompression supported: 1 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kLZ4Compression supported: 1 2024-04-27T02:28:41.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: kBZip2Compression supported: 0 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.merge_operator: 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_filter: None 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_filter_factory: None 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.sst_partitioner_factory: None 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561f69175ba0) 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-04-27T02:28:41.613 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 4 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x561f69179090 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2024-04-27T02:28:41.614 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: bloomfilter 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 5 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_auto_readahead_size: 262144 2024-04-27T02:28:41.615 INFO:journalctl@ceph.mon.c.smithi026.stdout: prepopulate_block_cache: 0 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout: initial_auto_readahead_size: 8192 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.write_buffer_size: 33554432 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_write_buffer_number: 2 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression: NoCompression 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression: Disabled 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.prefix_extractor: nullptr 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.num_levels: 7 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-27T02:28:41.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.level: 32767 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.strategy: 0 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.enabled: false 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-27T02:28:41.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.target_file_size_base: 67108864 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-27T02:28:41.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.arena_block_size: 1048576 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.disable_auto_compactions: 0 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.table_properties_collectors: 2024-04-27T02:28:41.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.inplace_update_support: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.bloom_locality: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.max_successive_merges: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.paranoid_file_checks: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.force_consistency_checks: 1 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.report_bg_io_stats: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.ttl: 2592000 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.enable_blob_files: false 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.min_blob_size: 0 2024-04-27T02:28:41.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.blob_file_size: 268435456 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.blob_file_starting_level: 0 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32692, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fa36e8ec-5989-42b1-accd-5ebf9eb80a75 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184921391567, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184921400663, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1702504, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32698, "largest_seqno": 33131, "table_properties": {"data_size": 1699576, "index_size": 1402, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4826, "raw_average_key_size": 23, "raw_value_size": 1695000, "raw_average_value_size": 8391, "num_data_blocks": 63, "num_entries": 202, "num_filter_entries": 202, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714184921, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fa36e8ec-5989-42b1-accd-5ebf9eb80a75", "db_session_id": "JSPGKQGNFG1WJBIIK8DV", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-04-27T02:28:41.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714184921400768, "job": 1, "event": "recovery_finished"} 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561f6926a000 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: DB pointer 0x561f69254000 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T02:28:41.622 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.62 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: L6 1/0 10.44 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 2/0 12.07 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-27T02:28:41.623 INFO:journalctl@ceph.mon.c.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 137.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 137.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache BinnedLRUCache@0x561f69179090#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-27T02:28:41.624 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: starting mon.c rank 1 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???) e3 preinit fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???).mds e1 new map 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???).mds e1 print_map 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: e1 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: -1 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout: No filesystems configured 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???).osd e111 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-27T02:28:41.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:28:41.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:28:41.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-04-27T02:28:41.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 ceph-mon[87214]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-04-27T02:28:41.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 bash[87195]: 0757865e99087bf49c382685996c8705e2d4384080bfe5fc75b834781aaa247b 2024-04-27T02:28:41.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:41 smithi026 systemd[1]: Started Ceph mon.c for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:28:42.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:28:42] "GET /metrics HTTP/1.1" 200 38006 "" "Prometheus/2.43.0" 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: fsmap 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: overall HEALTH_OK 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: mon.c calling monitor election 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: mon.a calling monitor election 2024-04-27T02:28:43.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: mon.b calling monitor election 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: monmap epoch 4 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: last_changed 2024-04-27T02:28:41.907218+0000 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: created 2024-04-27T01:57:44.453030+0000 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: min_mon_release 19 (squid) 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: election_strategy: 1 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:43.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: fsmap 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: overall HEALTH_OK 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: fsmap 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:43.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: overall HEALTH_OK 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: mon.c calling monitor election 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: mon.a calling monitor election 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: mon.b calling monitor election 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: monmap epoch 4 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: last_changed 2024-04-27T02:28:41.907218+0000 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: created 2024-04-27T01:57:44.453030+0000 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: min_mon_release 19 (squid) 2024-04-27T02:28:43.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: election_strategy: 1 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: fsmap 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: overall HEALTH_OK 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:43.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:43.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-27T02:28:43.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:43.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:43.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: fsmap 2024-04-27T02:28:43.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:43.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-27T02:28:43.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: overall HEALTH_OK 2024-04-27T02:28:43.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: mon.c calling monitor election 2024-04-27T02:28:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: mon.a calling monitor election 2024-04-27T02:28:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: mon.b calling monitor election 2024-04-27T02:28:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-27T02:28:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-27T02:28:43.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-27T02:28:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-27T02:28:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: monmap epoch 4 2024-04-27T02:28:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: fsid 7a5462be-0439-11ef-bc93-c7b262605968 2024-04-27T02:28:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: last_changed 2024-04-27T02:28:41.907218+0000 2024-04-27T02:28:43.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: created 2024-04-27T01:57:44.453030+0000 2024-04-27T02:28:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: min_mon_release 19 (squid) 2024-04-27T02:28:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: election_strategy: 1 2024-04-27T02:28:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-27T02:28:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-27T02:28:43.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: 2: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-27T02:28:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: fsmap 2024-04-27T02:28:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: osdmap e111: 8 total, 8 up, 8 in 2024-04-27T02:28:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-27T02:28:43.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: overall HEALTH_OK 2024-04-27T02:28:43.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:43.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:43.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:28:44.328 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:28:44.984 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:44 smithi026 ceph-mon[82260]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:44.984 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:44 smithi026 ceph-mon[82260]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:44.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:44 smithi026 ceph-mon[87214]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:44.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:44 smithi026 ceph-mon[87214]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 14s ago 24m 16.2M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (2m) 14s ago 24m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (3m) 14s ago 23m 53.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (3m) 14s ago 29m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (13m) 14s ago 30m 475M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (28s) 14s ago 30m 31.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (77s) 14s ago 29m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:28:45.025 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 starting - - - 2048M 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 14s ago 25m 8866k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (3m) 14s ago 25m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (28m) 14s ago 28m 54.8M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (28m) 14s ago 28m 55.9M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (28m) 14s ago 28m 51.0M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (27m) 14s ago 27m 56.6M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (27m) 14s ago 27m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (26m) 14s ago 26m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (26m) 14s ago 26m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (26m) 14s ago 26m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (2m) 14s ago 25m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:28:45.026 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (23m) 14s ago 23m 77.1M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:28:45.027 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (23m) 14s ago 23m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:28:45.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:44 smithi151 ceph-mon[64897]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:45.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:44 smithi151 ceph-mon[64897]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:45.446 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:28:45.446 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:28:45.447 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:28:45.448 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-27T02:28:45.448 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 5 2024-04-27T02:28:45.448 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:28:45.448 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:28:45.830 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:28:45.830 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:28:45.830 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:28:45.830 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi026", 2024-04-27T02:28:45.831 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:28:45.831 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "1/2 daemons upgraded", 2024-04-27T02:28:45.831 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-04-27T02:28:45.831 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:28:45.831 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:28:46.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:45 smithi151 ceph-mon[64897]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:46.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:45 smithi151 ceph-mon[64897]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:46.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:45 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/3310396324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:46.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:45 smithi026 ceph-mon[82260]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:46.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:45 smithi026 ceph-mon[82260]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:46.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:45 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/3310396324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:46.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:45 smithi026 ceph-mon[87214]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:46.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:45 smithi026 ceph-mon[87214]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:46.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:45 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/3310396324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:28:46.763 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:46 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:28:46.409+0000 7fc7ce81b700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-27T02:28:47.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:47 smithi151 ceph-mon[64897]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:47.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:47 smithi151 ceph-mon[64897]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:47.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:47 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:47.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:47 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[87214]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[87214]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[82260]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[82260]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:47.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:47 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:48.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:48.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:48 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:48.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:48 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:48.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:48.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:48 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:48.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:48 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:48.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:48.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:48 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:48.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:48 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:49.730 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:49 smithi026 ceph-mon[82260]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:49.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:49 smithi026 ceph-mon[87214]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:49.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:49 smithi151 ceph-mon[64897]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:51.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:51.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:51.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:51.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:51.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:28:51.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:51.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:51.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:51.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:52.347 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:28:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:28:52] "GET /metrics HTTP/1.1" 200 38006 "" "Prometheus/2.43.0" 2024-04-27T02:28:52.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:52 smithi026 ceph-mon[82260]: Reconfiguring mon.a (monmap changed)... 2024-04-27T02:28:52.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:52 smithi026 ceph-mon[82260]: Reconfiguring daemon mon.a on smithi026 2024-04-27T02:28:52.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:52 smithi026 ceph-mon[87214]: Reconfiguring mon.a (monmap changed)... 2024-04-27T02:28:52.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:52 smithi026 ceph-mon[87214]: Reconfiguring daemon mon.a on smithi026 2024-04-27T02:28:52.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:52 smithi151 ceph-mon[64897]: Reconfiguring mon.a (monmap changed)... 2024-04-27T02:28:52.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:52 smithi151 ceph-mon[64897]: Reconfiguring daemon mon.a on smithi026 2024-04-27T02:28:53.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[82260]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:53.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:53.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:53.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:28:53.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:28:53.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:53.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[87214]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:53.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:53.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:53.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:28:53.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:28:53.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:53.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:53 smithi151 ceph-mon[64897]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:53.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:53.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:53.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:28:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:28:53.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:54.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:54 smithi026 ceph-mon[82260]: Reconfiguring mgr.y (monmap changed)... 2024-04-27T02:28:54.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:54 smithi026 ceph-mon[82260]: Reconfiguring daemon mgr.y on smithi026 2024-04-27T02:28:54.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:54 smithi026 ceph-mon[87214]: Reconfiguring mgr.y (monmap changed)... 2024-04-27T02:28:54.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:54 smithi026 ceph-mon[87214]: Reconfiguring daemon mgr.y on smithi026 2024-04-27T02:28:54.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:54 smithi151 ceph-mon[64897]: Reconfiguring mgr.y (monmap changed)... 2024-04-27T02:28:54.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:54 smithi151 ceph-mon[64897]: Reconfiguring daemon mgr.y on smithi026 2024-04-27T02:28:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[82260]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[87214]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:55 smithi151 ceph-mon[64897]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:28:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:28:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:56.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[82260]: Reconfiguring mon.c (monmap changed)... 2024-04-27T02:28:56.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[82260]: Reconfiguring daemon mon.c on smithi026 2024-04-27T02:28:56.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:56.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:56.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:56.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[87214]: Reconfiguring mon.c (monmap changed)... 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[87214]: Reconfiguring daemon mon.c on smithi026 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T02:28:56.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:56.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:56 smithi151 ceph-mon[64897]: Reconfiguring mon.c (monmap changed)... 2024-04-27T02:28:56.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:56 smithi151 ceph-mon[64897]: Reconfiguring daemon mon.c on smithi026 2024-04-27T02:28:56.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:28:56.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:56.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:56.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T02:28:56.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:57.759 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:57 smithi151 ceph-mon[64897]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:57.759 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:57 smithi151 ceph-mon[64897]: Reconfiguring osd.0 (monmap changed)... 2024-04-27T02:28:57.759 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:57 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.0 on smithi026 2024-04-27T02:28:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:57 smithi026 ceph-mon[82260]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:57 smithi026 ceph-mon[82260]: Reconfiguring osd.0 (monmap changed)... 2024-04-27T02:28:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:57 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.0 on smithi026 2024-04-27T02:28:57.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:57 smithi026 ceph-mon[87214]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:28:57.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:57 smithi026 ceph-mon[87214]: Reconfiguring osd.0 (monmap changed)... 2024-04-27T02:28:57.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:57 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.0 on smithi026 2024-04-27T02:28:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:28:58.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:58 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:58.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:58 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:28:58.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:58 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:58.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:58 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:28:58.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:58 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:28:58.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:58 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:28:59.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:59 smithi151 ceph-mon[64897]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:59.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:59 smithi151 ceph-mon[64897]: Reconfiguring osd.1 (monmap changed)... 2024-04-27T02:28:59.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:28:59 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.1 on smithi026 2024-04-27T02:28:59.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:59 smithi026 ceph-mon[82260]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:59.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:59 smithi026 ceph-mon[82260]: Reconfiguring osd.1 (monmap changed)... 2024-04-27T02:28:59.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:28:59 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.1 on smithi026 2024-04-27T02:28:59.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:59 smithi026 ceph-mon[87214]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:28:59.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:59 smithi026 ceph-mon[87214]: Reconfiguring osd.1 (monmap changed)... 2024-04-27T02:28:59.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:28:59 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.1 on smithi026 2024-04-27T02:29:00.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:00.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:00.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[82260]: Reconfiguring osd.2 (monmap changed)... 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.2 on smithi026 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[87214]: Reconfiguring osd.2 (monmap changed)... 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:00.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:00 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.2 on smithi026 2024-04-27T02:29:00.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:00.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:00.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:00 smithi151 ceph-mon[64897]: Reconfiguring osd.2 (monmap changed)... 2024-04-27T02:29:00.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:29:00.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:00.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:00 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.2 on smithi026 2024-04-27T02:29:01.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[87214]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:01.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:01.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:01.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:29:01.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:01.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[82260]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:01.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:01.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:01.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:29:01.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:01 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:01.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:01 smithi151 ceph-mon[64897]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:01.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:01 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:01.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:01 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:01.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:01 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:29:01.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:01 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:02.533 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:29:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:29:02] "GET /metrics HTTP/1.1" 200 38020 "" "Prometheus/2.43.0" 2024-04-27T02:29:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:02 smithi151 ceph-mon[64897]: Reconfiguring osd.3 (monmap changed)... 2024-04-27T02:29:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:02 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.3 on smithi026 2024-04-27T02:29:02.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:02 smithi026 ceph-mon[82260]: Reconfiguring osd.3 (monmap changed)... 2024-04-27T02:29:02.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:02 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.3 on smithi026 2024-04-27T02:29:02.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:02 smithi026 ceph-mon[87214]: Reconfiguring osd.3 (monmap changed)... 2024-04-27T02:29:02.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:02 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.3 on smithi026 2024-04-27T02:29:03.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[82260]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:03.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:03.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:03.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:29:03.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:03.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[87214]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:03.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:03.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:03.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:29:03.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:03 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:03.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:03 smithi151 ceph-mon[64897]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:03.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:03 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:03.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:03 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:03.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:03 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:29:03.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:03 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:04 smithi151 ceph-mon[64897]: Reconfiguring rgw.foo.smithi026.oulbnt (monmap changed)... 2024-04-27T02:29:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:04 smithi151 ceph-mon[64897]: Reconfiguring daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:29:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:29:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:29:04.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:04.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[82260]: Reconfiguring rgw.foo.smithi026.oulbnt (monmap changed)... 2024-04-27T02:29:04.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[82260]: Reconfiguring daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:29:04.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:04.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:04.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:29:04.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[87214]: Reconfiguring rgw.foo.smithi026.oulbnt (monmap changed)... 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[87214]: Reconfiguring daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-27T02:29:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:05.841 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:05 smithi151 ceph-mon[64897]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:05.841 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:05 smithi151 ceph-mon[64897]: Reconfiguring mon.b (monmap changed)... 2024-04-27T02:29:05.841 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:05 smithi151 ceph-mon[64897]: Reconfiguring daemon mon.b on smithi151 2024-04-27T02:29:05.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:05 smithi026 ceph-mon[87214]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:05.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:05 smithi026 ceph-mon[87214]: Reconfiguring mon.b (monmap changed)... 2024-04-27T02:29:05.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:05 smithi026 ceph-mon[87214]: Reconfiguring daemon mon.b on smithi151 2024-04-27T02:29:05.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:05 smithi026 ceph-mon[82260]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:05.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:05 smithi026 ceph-mon[82260]: Reconfiguring mon.b (monmap changed)... 2024-04-27T02:29:05.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:05 smithi026 ceph-mon[82260]: Reconfiguring daemon mon.b on smithi151 2024-04-27T02:29:07.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:07.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:07.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:07.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: Reconfiguring mgr.x (monmap changed)... 2024-04-27T02:29:07.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:29:07.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:29:07.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:07.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:06 smithi151 ceph-mon[64897]: Reconfiguring daemon mgr.x on smithi151 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: Reconfiguring mgr.x (monmap changed)... 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:07.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[82260]: Reconfiguring daemon mgr.x on smithi151 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: Reconfiguring mgr.x (monmap changed)... 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:07.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:06 smithi026 ceph-mon[87214]: Reconfiguring daemon mgr.x on smithi151 2024-04-27T02:29:08.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:08 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:08.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:08 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:08.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:08 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:08.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:08 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:29:08.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:08 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:08.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:08.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:29:08.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:08 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:09 smithi151 ceph-mon[64897]: Reconfiguring osd.4 (monmap changed)... 2024-04-27T02:29:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:09 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.4 on smithi151 2024-04-27T02:29:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:09 smithi151 ceph-mon[64897]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:29:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:09.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[82260]: Reconfiguring osd.4 (monmap changed)... 2024-04-27T02:29:09.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.4 on smithi151 2024-04-27T02:29:09.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[82260]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:09.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:09.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:09.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:29:09.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:09.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[87214]: Reconfiguring osd.4 (monmap changed)... 2024-04-27T02:29:09.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.4 on smithi151 2024-04-27T02:29:09.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[87214]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:09.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:09.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:09.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:29:09.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:10.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:10 smithi151 ceph-mon[64897]: Reconfiguring osd.5 (monmap changed)... 2024-04-27T02:29:10.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:10 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.5 on smithi151 2024-04-27T02:29:10.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:10 smithi026 ceph-mon[87214]: Reconfiguring osd.5 (monmap changed)... 2024-04-27T02:29:10.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:10 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.5 on smithi151 2024-04-27T02:29:10.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:10 smithi026 ceph-mon[82260]: Reconfiguring osd.5 (monmap changed)... 2024-04-27T02:29:10.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:10 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.5 on smithi151 2024-04-27T02:29:11.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:11 smithi151 ceph-mon[64897]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:11.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:11.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:11.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:29:11.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:11.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[87214]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:11.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:11.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:29:11.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:11.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:11.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[82260]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:11.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:11.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:11.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:29:11.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:11.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:12.562 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:29:12] "GET /metrics HTTP/1.1" 200 38017 "" "Prometheus/2.43.0" 2024-04-27T02:29:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:12 smithi151 ceph-mon[64897]: Reconfiguring osd.6 (monmap changed)... 2024-04-27T02:29:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:12 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.6 on smithi151 2024-04-27T02:29:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:12 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:12 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:12 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:29:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:12 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[82260]: Reconfiguring osd.6 (monmap changed)... 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.6 on smithi151 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[87214]: Reconfiguring osd.6 (monmap changed)... 2024-04-27T02:29:12.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.6 on smithi151 2024-04-27T02:29:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:29:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:12 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:13.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:13 smithi151 ceph-mon[64897]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:13.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:13 smithi151 ceph-mon[64897]: Reconfiguring osd.7 (monmap changed)... 2024-04-27T02:29:13.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:13 smithi151 ceph-mon[64897]: Reconfiguring daemon osd.7 on smithi151 2024-04-27T02:29:13.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:13 smithi026 ceph-mon[82260]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:13.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:13 smithi026 ceph-mon[82260]: Reconfiguring osd.7 (monmap changed)... 2024-04-27T02:29:13.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:13 smithi026 ceph-mon[82260]: Reconfiguring daemon osd.7 on smithi151 2024-04-27T02:29:13.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:13 smithi026 ceph-mon[87214]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:13.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:13 smithi026 ceph-mon[87214]: Reconfiguring osd.7 (monmap changed)... 2024-04-27T02:29:13.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:13 smithi026 ceph-mon[87214]: Reconfiguring daemon osd.7 on smithi151 2024-04-27T02:29:15.266 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:15 smithi151 ceph-mon[64897]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:15.267 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:15.267 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:15.267 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:15 smithi151 ceph-mon[64897]: Reconfiguring rgw.foo.smithi151.blhads (monmap changed)... 2024-04-27T02:29:15.267 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:29:15.267 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:15.267 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:15 smithi151 ceph-mon[64897]: Reconfiguring daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:29:15.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[82260]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:15.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:15.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:15.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[82260]: Reconfiguring rgw.foo.smithi151.blhads (monmap changed)... 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[82260]: Reconfiguring daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[87214]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[87214]: Reconfiguring rgw.foo.smithi151.blhads (monmap changed)... 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:15 smithi026 ceph-mon[87214]: Reconfiguring daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:29:16.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mon 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-27T02:29:16.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all crash 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mds 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:29:16.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nfs 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mon 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all crash 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mds 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:29:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nfs 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all node-exporter 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all prometheus 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all alertmanager 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all grafana 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all loki 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all promtail 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Finalizing container_image settings 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all node-exporter 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all prometheus 2024-04-27T02:29:16.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all alertmanager 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all grafana 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all loki 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all promtail 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Finalizing container_image settings 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:29:16.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:29:16.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:29:16.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:29:16.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: Upgrade: Complete! 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:29:16.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[87214]: from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: Upgrade: Complete! 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:16.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:16 smithi026 ceph-mon[82260]: from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:16.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:29:17.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:17.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mon 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all crash 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mds 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:29:17.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nfs 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all node-exporter 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all prometheus 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all alertmanager 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all grafana 2024-04-27T02:29:17.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all loki 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all promtail 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Finalizing container_image settings 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:29:17.100 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: Upgrade: Complete! 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:29:17.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:17.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:16 smithi151 ceph-mon[64897]: from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:17.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:17 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:17.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:17 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:18.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:17 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:18.767 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:29:18.767 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 32s ago 25m 16.3M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:29:18.767 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (2m) 48s ago 24m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:29:18.767 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (3m) 32s ago 24m 56.6M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:29:18.767 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (4m) 48s ago 29m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (14m) 32s ago 31m 493M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (62s) 32s ago 31m 38.0M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (111s) 48s ago 29m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (37s) 32s ago 30m 20.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 32s ago 25m 8019k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (3m) 48s ago 25m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (29m) 32s ago 29m 55.2M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (29m) 32s ago 29m 56.0M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (28m) 32s ago 28m 51.0M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (28m) 32s ago 28m 57.0M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (27m) 48s ago 27m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (27m) 48s ago 27m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (27m) 48s ago 27m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (26m) 48s ago 26m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:29:18.768 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (3m) 48s ago 26m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:29:18.769 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (24m) 32s ago 24m 77.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:29:18.769 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (24m) 48s ago 24m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:29:19.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:18 smithi026 ceph-mon[82260]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:19.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:18 smithi026 ceph-mon[87214]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:19.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:18 smithi151 ceph-mon[64897]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:19.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-04-27T02:29:19.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:19 smithi026 ceph-mon[82260]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:19.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:19 smithi026 ceph-mon[87214]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:20.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:19 smithi151 ceph-mon[64897]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:20.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:20 smithi026 ceph-mon[82260]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:20.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:20 smithi026 ceph-mon[87214]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:20 smithi151 ceph-mon[64897]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:21.373 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:29:21.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-04-27T02:29:22.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:21 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2019305172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:22.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:21 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2019305172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:22.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:21 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2019305172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:22.407 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:29:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:29:22] "GET /metrics HTTP/1.1" 200 38017 "" "Prometheus/2.43.0" 2024-04-27T02:29:23.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:22 smithi151 ceph-mon[64897]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:23.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:22 smithi026 ceph-mon[82260]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:23.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:22 smithi026 ceph-mon[87214]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:23.940 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)" 2024-04-27T02:29:24.680 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-04-27T02:29:24.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:24 smithi026 ceph-mon[82260]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:24.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:24 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/4276145226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:24.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:24 smithi026 ceph-mon[87214]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:24.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:24 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/4276145226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:25.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:24 smithi151 ceph-mon[64897]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:25.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:24 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/4276145226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:26.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:26.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:26.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:27.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:26 smithi151 ceph-mon[64897]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:26 smithi026 ceph-mon[82260]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:27.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:26 smithi026 ceph-mon[87214]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:27.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:27 smithi026 ceph-mon[82260]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:27.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:27 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:27.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:27 smithi026 ceph-mon[87214]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:27.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:27 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:28.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:27 smithi151 ceph-mon[64897]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:28.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:27 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:28.803 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:29:29.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:28 smithi151 ceph-mon[64897]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:29.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:28 smithi026 ceph-mon[82260]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:29.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:28 smithi026 ceph-mon[87214]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:29.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:29:31.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:30 smithi026 ceph-mon[82260]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:31.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:30 smithi026 ceph-mon[87214]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:31.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:30 smithi151 ceph-mon[64897]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:31.590 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:29:31.591 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:29:32.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:29:32.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:29:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:29:32] "GET /metrics HTTP/1.1" 200 38016 "" "Prometheus/2.43.0" 2024-04-27T02:29:33.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:32 smithi026 ceph-mon[82260]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:33.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:32 smithi026 ceph-mon[82260]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:33.031 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:32 smithi026 ceph-mon[87214]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:33.031 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:32 smithi026 ceph-mon[87214]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:33.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:32 smithi151 ceph-mon[64897]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:33.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:32 smithi151 ceph-mon[64897]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:34.442 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:29:35.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:34 smithi026 ceph-mon[82260]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:35.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:34 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/128510344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:29:35.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:34 smithi026 ceph-mon[87214]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:35.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:34 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/128510344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:29:35.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:34 smithi151 ceph-mon[64897]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:35.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:34 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/128510344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:29:35.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-04-27T02:29:37.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:36 smithi026 ceph-mon[82260]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:37.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:36 smithi026 ceph-mon[87214]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:37.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:36 smithi151 ceph-mon[64897]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:38.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:38 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:38.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:38 smithi151 ceph-mon[64897]: from='client.54145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:38.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:38 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:38.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:38 smithi026 ceph-mon[82260]: from='client.54145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:38.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:38 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:38.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:38 smithi026 ceph-mon[87214]: from='client.54145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:39.299 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:29:39.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:39 smithi151 ceph-mon[64897]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:39.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:39 smithi026 ceph-mon[82260]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:39.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:39 smithi026 ceph-mon[87214]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:39.978 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-27T02:29:40.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:40 smithi151 ceph-mon[64897]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:29:40.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:40.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[82260]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:29:40.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:40.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[87214]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:40.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:41.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[82260]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:41.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:41.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:41.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:41.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[82260]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:29:41.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:41.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[87214]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:41.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:41.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:41.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:41.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[87214]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:29:41.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:41 smithi151 ceph-mon[64897]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:29:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:41 smithi151 ceph-mon[64897]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:29:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:42.016 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:29:42.317 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:29:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:29:42] "GET /metrics HTTP/1.1" 200 38015 "" "Prometheus/2.43.0" 2024-04-27T02:29:42.735 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:29:42.735 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 56s ago 25m 16.3M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (3m) 72s ago 25m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (4m) 56s ago 24m 56.6M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (4m) 72s ago 30m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (14m) 56s ago 31m 493M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (86s) 56s ago 31m 38.0M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (2m) 72s ago 30m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (61s) 56s ago 30m 20.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 56s ago 26m 8019k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (4m) 72s ago 26m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (29m) 56s ago 29m 55.2M 2993M 17.2.0 e1d6a67b021e b9ad186ce817 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (29m) 56s ago 29m 56.0M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (29m) 56s ago 29m 51.0M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (28m) 56s ago 28m 57.0M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:29:42.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (28m) 72s ago 28m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:29:42.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (27m) 72s ago 27m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:29:42.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (27m) 72s ago 27m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:29:42.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (27m) 72s ago 27m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:29:42.737 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (3m) 72s ago 26m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:29:42.737 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (24m) 56s ago 24m 77.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:29:42.737 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (24m) 72s ago 24m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:29:43.149 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:29:43.150 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:29:43.150 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:29:43.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:29:43.150 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:29:43.150 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:29:43.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:29:43.151 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:29:43.151 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-27T02:29:43.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:29:43.151 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:29:43.151 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:29:43.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:29:43.151 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:29:43.152 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-27T02:29:43.152 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 5 2024-04-27T02:29:43.152 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:29:43.152 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:29:43.509 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:29:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:29:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:29:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-04-27T02:29:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:29:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/8 daemons upgraded", 2024-04-27T02:29:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:29:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:29:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:29:43.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[82260]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:43.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[82260]: from='client.54151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:43.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[82260]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:43.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2676741718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:43.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[87214]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:43.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[87214]: from='client.54151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:43.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[87214]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:43.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:43 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2676741718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:43.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:43 smithi151 ceph-mon[64897]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:43.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:43 smithi151 ceph-mon[64897]: from='client.54151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:43.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:43 smithi151 ceph-mon[64897]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:43.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:43 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2676741718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:44.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:29:44.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:29:44.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:44.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mgr 2024-04-27T02:29:44.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mon 2024-04-27T02:29:44.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all crash 2024-04-27T02:29:44.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-27T02:29:44.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-27T02:29:44.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: Upgrade: osd.0 is safe to restart 2024-04-27T02:29:44.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[87214]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:44.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:44.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:29:44.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:29:44.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:44.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mgr 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mon 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all crash 2024-04-27T02:29:44.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-27T02:29:44.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-27T02:29:44.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: Upgrade: osd.0 is safe to restart 2024-04-27T02:29:44.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:44 smithi026 ceph-mon[82260]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:29:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mgr 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mon 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all crash 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-27T02:29:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: Upgrade: osd.0 is safe to restart 2024-04-27T02:29:44.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:44 smithi151 ceph-mon[64897]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:29:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:45 smithi151 ceph-mon[64897]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T02:29:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[87214]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[82260]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-27T02:29:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:29:46.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:46 smithi151 ceph-mon[64897]: Upgrade: Updating osd.0 2024-04-27T02:29:46.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:46 smithi151 ceph-mon[64897]: Deploying daemon osd.0 on smithi026 2024-04-27T02:29:46.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:46 smithi026 ceph-mon[82260]: Upgrade: Updating osd.0 2024-04-27T02:29:46.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:46 smithi026 ceph-mon[82260]: Deploying daemon osd.0 on smithi026 2024-04-27T02:29:46.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:46 smithi026 ceph-mon[87214]: Upgrade: Updating osd.0 2024-04-27T02:29:46.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:46 smithi026 ceph-mon[87214]: Deploying daemon osd.0 on smithi026 2024-04-27T02:29:47.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:47 smithi151 ceph-mon[64897]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:47.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:47 smithi026 ceph-mon[82260]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:47.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:47 smithi026 ceph-mon[87214]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:29:48.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:48.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:48.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:48.604 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:48 smithi026 systemd[1]: Stopping Ceph osd.0 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:29:49.103 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[29440]: 2024-04-27T02:29:48.663+0000 7fa151da1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:29:49.103 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[29440]: 2024-04-27T02:29:48.663+0000 7fa151da1700 -1 osd.0 111 *** Got signal Terminated *** 2024-04-27T02:29:49.103 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:48 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[29440]: 2024-04-27T02:29:48.663+0000 7fa151da1700 -1 osd.0 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:29:49.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:49 smithi026 ceph-mon[82260]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:49.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:49 smithi026 ceph-mon[82260]: osd.0 marked itself down and dead 2024-04-27T02:29:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:49 smithi026 ceph-mon[87214]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:49 smithi026 ceph-mon[87214]: osd.0 marked itself down and dead 2024-04-27T02:29:49.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:49 smithi151 ceph-mon[64897]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:29:49.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:49 smithi151 ceph-mon[64897]: osd.0 marked itself down and dead 2024-04-27T02:29:50.140 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:49 smithi026 podman[94159]: 2024-04-27 02:29:49.880263507 +0000 UTC m=+1.347149281 container died b9ad186ce817c92279f03f4597cf726e549385b037a372b6fc5841ef00dad094 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, architecture=x86_64, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream) 2024-04-27T02:29:50.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:50 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:29:50.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:50 smithi026 ceph-mon[82260]: osdmap e112: 8 total, 7 up, 8 in 2024-04-27T02:29:50.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:50 smithi026 ceph-mon[82260]: osdmap e113: 8 total, 7 up, 8 in 2024-04-27T02:29:50.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:50 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:29:50.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:50 smithi026 ceph-mon[87214]: osdmap e112: 8 total, 7 up, 8 in 2024-04-27T02:29:50.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:50 smithi026 ceph-mon[87214]: osdmap e113: 8 total, 7 up, 8 in 2024-04-27T02:29:50.481 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:50 smithi026 podman[94159]: 2024-04-27 02:29:50.254910165 +0000 UTC m=+1.721795927 container cleanup b9ad186ce817c92279f03f4597cf726e549385b037a372b6fc5841ef00dad094 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., ceph=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, RELEASE=HEAD, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-type=git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-04-27T02:29:50.481 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:50 smithi026 bash[94159]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0 2024-04-27T02:29:50.814 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:50 smithi026 podman[94178]: 2024-04-27 02:29:50.50643307 +0000 UTC m=+0.626307173 container remove b9ad186ce817c92279f03f4597cf726e549385b037a372b6fc5841ef00dad094 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, com.redhat.component=centos-stream-container, vcs-type=git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, architecture=x86_64, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-27T02:29:50.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:50 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:29:50.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:50 smithi151 ceph-mon[64897]: osdmap e112: 8 total, 7 up, 8 in 2024-04-27T02:29:50.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:50 smithi151 ceph-mon[64897]: osdmap e113: 8 total, 7 up, 8 in 2024-04-27T02:29:51.076 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:50 smithi026 podman[94270]: 2024-04-27 02:29:50.840775201 +0000 UTC m=+0.115631391 container create 690e6de65b7917bd4f2fb9040ba5cc494687ee7adb19f32bd6cf8abb6803ceb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:29:51.077 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:50 smithi026 podman[94270]: 2024-04-27 02:29:50.743736143 +0000 UTC m=+0.018592356 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:29:51.077 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:51 smithi026 podman[94270]: 2024-04-27 02:29:51.102092462 +0000 UTC m=+0.376948661 container init 690e6de65b7917bd4f2fb9040ba5cc494687ee7adb19f32bd6cf8abb6803ceb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:29:51.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:51 smithi026 podman[94270]: 2024-04-27 02:29:51.106072871 +0000 UTC m=+0.380929064 container start 690e6de65b7917bd4f2fb9040ba5cc494687ee7adb19f32bd6cf8abb6803ceb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-27T02:29:51.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:51 smithi026 podman[94270]: 2024-04-27 02:29:51.210146131 +0000 UTC m=+0.485002335 container attach 690e6de65b7917bd4f2fb9040ba5cc494687ee7adb19f32bd6cf8abb6803ceb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=) 2024-04-27T02:29:51.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:51 smithi026 ceph-mon[82260]: pgmap v46: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:29:51.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:51 smithi026 ceph-mon[87214]: pgmap v46: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:29:51.705 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:51 smithi026 podman[94270]: 2024-04-27 02:29:51.408933997 +0000 UTC m=+0.683790202 container died 690e6de65b7917bd4f2fb9040ba5cc494687ee7adb19f32bd6cf8abb6803ceb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=) 2024-04-27T02:29:51.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:51 smithi151 ceph-mon[64897]: pgmap v46: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:29:52.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:29:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:29:52] "GET /metrics HTTP/1.1" 200 38015 "" "Prometheus/2.43.0" 2024-04-27T02:29:52.236 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:51 smithi026 podman[94296]: 2024-04-27 02:29:51.997323965 +0000 UTC m=+0.582633408 container remove 690e6de65b7917bd4f2fb9040ba5cc494687ee7adb19f32bd6cf8abb6803ceb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-27T02:29:52.236 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-27T02:29:52.236 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 systemd[1]: Stopped Ceph osd.0 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:29:52.236 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.0.service: Consumed 15.337s CPU time. 2024-04-27T02:29:52.236 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 systemd[1]: Starting Ceph osd.0 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:29:52.603 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 podman[94397]: 2024-04-27 02:29:52.363501388 +0000 UTC m=+0.019562146 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:29:52.603 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 podman[94397]: 2024-04-27 02:29:52.507534063 +0000 UTC m=+0.163594821 container create 2eaaba148b2581959fa1bc9d2744c4c2471a34cc872a2ecaa83069994595bc87 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-27T02:29:53.089 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 podman[94397]: 2024-04-27 02:29:52.726980685 +0000 UTC m=+0.383041446 container init 2eaaba148b2581959fa1bc9d2744c4c2471a34cc872a2ecaa83069994595bc87 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-27T02:29:53.089 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 podman[94397]: 2024-04-27 02:29:52.730367541 +0000 UTC m=+0.386428300 container start 2eaaba148b2581959fa1bc9d2744c4c2471a34cc872a2ecaa83069994595bc87 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-27T02:29:53.089 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:52 smithi026 podman[94397]: 2024-04-27 02:29:52.827641171 +0000 UTC m=+0.483701948 container attach 2eaaba148b2581959fa1bc9d2744c4c2471a34cc872a2ecaa83069994595bc87 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-27T02:29:53.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:53 smithi151 ceph-mon[64897]: pgmap v48: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:53.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:53 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded (PG_DEGRADED) 2024-04-27T02:29:53.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-mon[82260]: pgmap v48: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:53.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded (PG_DEGRADED) 2024-04-27T02:29:53.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-mon[87214]: pgmap v48: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:53.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded (PG_DEGRADED) 2024-04-27T02:29:53.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate[94418]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate[94418]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 bash[94397]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 bash[94397]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 bash[94397]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 bash[94397]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 bash[94397]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 bash[94397]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 bash[94397]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate[94418]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate[94418]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate[94418]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate[94418]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-27T02:29:53.354 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate[94418]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-27T02:29:53.355 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 podman[94397]: 2024-04-27 02:29:53.216198059 +0000 UTC m=+0.872258812 container died 2eaaba148b2581959fa1bc9d2744c4c2471a34cc872a2ecaa83069994595bc87 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-27T02:29:54.103 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:53 smithi026 podman[94547]: 2024-04-27 02:29:53.830752085 +0000 UTC m=+0.608647826 container remove 2eaaba148b2581959fa1bc9d2744c4c2471a34cc872a2ecaa83069994595bc87 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-activate, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-27T02:29:54.103 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 podman[94615]: 2024-04-27 02:29:54.040400654 +0000 UTC m=+0.106611165 container create 860b174a16df4bb6eba3e8c018be6a448dd8a57ba7d68f7be72b38bcd6348771 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-27T02:29:54.103 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 podman[94615]: 2024-04-27 02:29:53.952417162 +0000 UTC m=+0.018627679 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:29:54.392 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 podman[94615]: 2024-04-27 02:29:54.317914699 +0000 UTC m=+0.384125215 container init 860b174a16df4bb6eba3e8c018be6a448dd8a57ba7d68f7be72b38bcd6348771 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:29:54.392 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 podman[94615]: 2024-04-27 02:29:54.321184502 +0000 UTC m=+0.387395013 container start 860b174a16df4bb6eba3e8c018be6a448dd8a57ba7d68f7be72b38bcd6348771 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-27T02:29:54.392 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 ceph-osd[94633]: -- 172.21.15.26:0/990351918 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bd1bf25680 con 0x55bd1beef000 2024-04-27T02:29:54.838 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 bash[94615]: 860b174a16df4bb6eba3e8c018be6a448dd8a57ba7d68f7be72b38bcd6348771 2024-04-27T02:29:54.838 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 systemd[1]: Started Ceph osd.0 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:29:55.103 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[94629]: 2024-04-27T02:29:54.863+0000 7f7a210687c0 -1 Falling back to public interface 2024-04-27T02:29:55.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[82260]: pgmap v49: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:55.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:55.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[87214]: pgmap v49: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:55.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:55 smithi151 ceph-mon[64897]: pgmap v49: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:55.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:29:57.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:56 smithi026 ceph-mon[82260]: pgmap v50: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:57.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:57.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:57.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:56 smithi026 ceph-mon[87214]: pgmap v50: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:57.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:57.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:56 smithi151 ceph-mon[64897]: pgmap v50: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:57.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:29:57.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:29:57.817 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[94629]: 2024-04-27T02:29:57.572+0000 7f7a210687c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-27T02:29:57.818 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[94629]: 2024-04-27T02:29:57.604+0000 7f7a210687c0 -1 osd.0 111 log_to_monitors true 2024-04-27T02:29:58.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:57 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:58.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:57 smithi151 ceph-mon[64897]: from='osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T02:29:58.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:57 smithi151 ceph-mon[64897]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T02:29:58.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:58.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-mon[82260]: from='osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T02:29:58.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-mon[82260]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T02:29:58.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:29:58.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-mon[87214]: from='osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T02:29:58.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:57 smithi026 ceph-mon[87214]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-27T02:29:58.459 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[94629]: 2024-04-27T02:29:58.133+0000 7f7a0e4b8700 -1 osd.0 111 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:29:59.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:58 smithi151 ceph-mon[64897]: pgmap v51: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:59.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:58 smithi151 ceph-mon[64897]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-27T02:29:59.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:58 smithi151 ceph-mon[64897]: osdmap e114: 8 total, 7 up, 8 in 2024-04-27T02:29:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:58 smithi151 ceph-mon[64897]: from='osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:29:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:58 smithi151 ceph-mon[64897]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:29:59.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:58 smithi151 ceph-mon[64897]: from='osd.0 ' entity='osd.0' 2024-04-27T02:29:59.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[87214]: pgmap v51: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:59.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[87214]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-27T02:29:59.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[87214]: osdmap e114: 8 total, 7 up, 8 in 2024-04-27T02:29:59.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[87214]: from='osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:29:59.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[87214]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:29:59.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[87214]: from='osd.0 ' entity='osd.0' 2024-04-27T02:29:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[82260]: pgmap v51: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-04-27T02:29:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[82260]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-27T02:29:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[82260]: osdmap e114: 8 total, 7 up, 8 in 2024-04-27T02:29:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[82260]: from='osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:29:59.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[82260]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:29:59.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:58 smithi026 ceph-mon[82260]: from='osd.0 ' entity='osd.0' 2024-04-27T02:30:00.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:59 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:59 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:59 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:30:00.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:59 smithi151 ceph-mon[64897]: osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318] boot 2024-04-27T02:30:00.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:59 smithi151 ceph-mon[64897]: osdmap e115: 8 total, 8 up, 8 in 2024-04-27T02:30:00.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:29:59 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:30:00.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:30:00.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[82260]: osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318] boot 2024-04-27T02:30:00.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[82260]: osdmap e115: 8 total, 8 up, 8 in 2024-04-27T02:30:00.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:30:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:30:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[87214]: osd.0 [v2:172.21.15.26:6802/2061502318,v1:172.21.15.26:6803/2061502318] boot 2024-04-27T02:30:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[87214]: osdmap e115: 8 total, 8 up, 8 in 2024-04-27T02:30:00.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:29:59 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-27T02:30:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pgmap v54: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: Health detail: HEALTH_WARN Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: [WRN] PG_DEGRADED: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 2.f is active+undersized+degraded, acting [4,2] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-27T02:30:00.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: pg 5.1b is active+undersized+degraded, acting [5,7] 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[82260]: osdmap e116: 8 total, 8 up, 8 in 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pgmap v54: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-04-27T02:30:00.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: Health detail: HEALTH_WARN Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: [WRN] PG_DEGRADED: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 2.f is active+undersized+degraded, acting [4,2] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-04-27T02:30:00.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: pg 5.1b is active+undersized+degraded, acting [5,7] 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:00.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:00 smithi026 ceph-mon[87214]: osdmap e116: 8 total, 8 up, 8 in 2024-04-27T02:30:01.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pgmap v54: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-04-27T02:30:01.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: Health detail: HEALTH_WARN Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded 2024-04-27T02:30:01.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: [WRN] PG_DEGRADED: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 2.f is active+undersized+degraded, acting [4,2] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-04-27T02:30:01.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: pg 5.1b is active+undersized+degraded, acting [5,7] 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:01.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:00 smithi151 ceph-mon[64897]: osdmap e116: 8 total, 8 up, 8 in 2024-04-27T02:30:02.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:01 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:02.215 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:01 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:02.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:01 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:02.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:30:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:30:02] "GET /metrics HTTP/1.1" 200 38148 "" "Prometheus/2.43.0" 2024-04-27T02:30:03.242 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:02 smithi026 ceph-mon[82260]: pgmap v56: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-04-27T02:30:03.243 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:02 smithi026 ceph-mon[87214]: pgmap v56: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-04-27T02:30:03.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:02 smithi151 ceph-mon[64897]: pgmap v56: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[87214]: Upgrade: osd.1 is safe to restart 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:04.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:04.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:04.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-27T02:30:04.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-27T02:30:04.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:04 smithi026 ceph-mon[82260]: Upgrade: osd.1 is safe to restart 2024-04-27T02:30:05.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:05.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:05.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:05.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:05.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-27T02:30:05.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:04 smithi151 ceph-mon[64897]: Upgrade: osd.1 is safe to restart 2024-04-27T02:30:05.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded) 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[82260]: Upgrade: Updating osd.1 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[82260]: Deploying daemon osd.1 on smithi026 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded) 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[87214]: Upgrade: Updating osd.1 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:05.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:05 smithi026 ceph-mon[87214]: Deploying daemon osd.1 on smithi026 2024-04-27T02:30:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:05 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded) 2024-04-27T02:30:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:05 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:30:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:05 smithi151 ceph-mon[64897]: Upgrade: Updating osd.1 2024-04-27T02:30:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-27T02:30:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:06.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:05 smithi151 ceph-mon[64897]: Deploying daemon osd.1 on smithi026 2024-04-27T02:30:06.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:06 smithi026 ceph-mon[82260]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 644 B/s rd, 0 op/s 2024-04-27T02:30:06.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:06 smithi026 ceph-mon[87214]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 644 B/s rd, 0 op/s 2024-04-27T02:30:07.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:06 smithi151 ceph-mon[64897]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 644 B/s rd, 0 op/s 2024-04-27T02:30:07.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:07 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:07.998 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:07 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:08.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:07 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:09.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:08 smithi151 ceph-mon[64897]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:09.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:08 smithi026 ceph-mon[82260]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:09.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:08 smithi026 ceph-mon[87214]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:09.103 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:08 smithi026 systemd[1]: Stopping Ceph osd.1 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:30:09.103 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[32407]: 2024-04-27T02:30:08.887+0000 7fd1206f2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:30:09.104 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[32407]: 2024-04-27T02:30:08.887+0000 7fd1206f2700 -1 osd.1 116 *** Got signal Terminated *** 2024-04-27T02:30:09.104 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[32407]: 2024-04-27T02:30:08.887+0000 7fd1206f2700 -1 osd.1 116 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:30:10.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:09 smithi151 ceph-mon[64897]: osd.1 marked itself down and dead 2024-04-27T02:30:10.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:09 smithi026 ceph-mon[82260]: osd.1 marked itself down and dead 2024-04-27T02:30:10.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:09 smithi026 ceph-mon[87214]: osd.1 marked itself down and dead 2024-04-27T02:30:10.505 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:10 smithi026 podman[99260]: 2024-04-27 02:30:10.219735001 +0000 UTC m=+1.402135491 container died 8005748a9388c0d1c4255bd889f24815c9247e17ffa377a11c29c413b2afff8e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-type=git, maintainer=Guillaume Abrioux , version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.component=centos-stream-container, distribution-scope=public) 2024-04-27T02:30:10.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:10 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:30:10.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:10 smithi026 ceph-mon[87214]: osdmap e117: 8 total, 7 up, 8 in 2024-04-27T02:30:10.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:10 smithi026 ceph-mon[87214]: pgmap v61: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:30:10.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:10 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:30:10.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:10 smithi026 ceph-mon[82260]: osdmap e117: 8 total, 7 up, 8 in 2024-04-27T02:30:10.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:10 smithi026 ceph-mon[82260]: pgmap v61: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:30:10.854 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:10 smithi026 podman[99260]: 2024-04-27 02:30:10.629206856 +0000 UTC m=+1.811607357 container cleanup 8005748a9388c0d1c4255bd889f24815c9247e17ffa377a11c29c413b2afff8e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, architecture=x86_64, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., ceph=True, name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8) 2024-04-27T02:30:10.854 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:10 smithi026 bash[99260]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1 2024-04-27T02:30:11.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:10 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:30:11.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:10 smithi151 ceph-mon[64897]: osdmap e117: 8 total, 7 up, 8 in 2024-04-27T02:30:11.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:10 smithi151 ceph-mon[64897]: pgmap v61: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:30:11.280 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:10 smithi026 podman[99278]: 2024-04-27 02:30:10.880841286 +0000 UTC m=+0.663168346 container remove 8005748a9388c0d1c4255bd889f24815c9247e17ffa377a11c29c413b2afff8e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-type=git, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, version=8) 2024-04-27T02:30:11.558 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:11 smithi026 podman[99370]: 2024-04-27 02:30:11.209478063 +0000 UTC m=+0.018781775 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:30:11.559 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:11 smithi026 podman[99370]: 2024-04-27 02:30:11.306761611 +0000 UTC m=+0.116065313 container create d0e09f30a7a99460bcb7892f5dcae8a25692d5fc11d5fc9b8398660cd5e6e532 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-b22e2eb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[87214]: osdmap e118: 8 total, 7 up, 8 in 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[82260]: osdmap e118: 8 total, 7 up, 8 in 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:11.854 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:11 smithi026 podman[99370]: 2024-04-27 02:30:11.584542776 +0000 UTC m=+0.393846478 container init d0e09f30a7a99460bcb7892f5dcae8a25692d5fc11d5fc9b8398660cd5e6e532 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-27T02:30:11.854 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:11 smithi026 podman[99370]: 2024-04-27 02:30:11.588063975 +0000 UTC m=+0.397367678 container start d0e09f30a7a99460bcb7892f5dcae8a25692d5fc11d5fc9b8398660cd5e6e532 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb) 2024-04-27T02:30:11.854 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:11 smithi026 podman[99370]: 2024-04-27 02:30:11.734581855 +0000 UTC m=+0.543885568 container attach d0e09f30a7a99460bcb7892f5dcae8a25692d5fc11d5fc9b8398660cd5e6e532 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-27T02:30:12.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:11 smithi151 ceph-mon[64897]: osdmap e118: 8 total, 7 up, 8 in 2024-04-27T02:30:12.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:12.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:12.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:12.214 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:11 smithi026 podman[99370]: 2024-04-27 02:30:11.886978534 +0000 UTC m=+0.696282231 container died d0e09f30a7a99460bcb7892f5dcae8a25692d5fc11d5fc9b8398660cd5e6e532 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=) 2024-04-27T02:30:12.465 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:12 smithi026 podman[99395]: 2024-04-27 02:30:12.4883216 +0000 UTC m=+0.596150667 container remove d0e09f30a7a99460bcb7892f5dcae8a25692d5fc11d5fc9b8398660cd5e6e532 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb) 2024-04-27T02:30:12.465 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:30:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:30:12] "GET /metrics HTTP/1.1" 200 38159 "" "Prometheus/2.43.0" 2024-04-27T02:30:12.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:12 smithi026 ceph-mon[82260]: pgmap v63: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-27T02:30:12.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:12 smithi026 ceph-mon[87214]: pgmap v63: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-27T02:30:12.854 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:12 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-27T02:30:12.854 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:12 smithi026 systemd[1]: Stopped Ceph osd.1 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:30:12.855 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:12 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.1.service: Consumed 16.016s CPU time. 2024-04-27T02:30:12.855 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:12 smithi026 systemd[1]: Starting Ceph osd.1 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:30:13.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:12 smithi151 ceph-mon[64897]: pgmap v63: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-27T02:30:13.234 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:12 smithi026 podman[99500]: 2024-04-27 02:30:12.847396974 +0000 UTC m=+0.018357181 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:30:13.234 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:12 smithi026 podman[99500]: 2024-04-27 02:30:12.964829457 +0000 UTC m=+0.135789648 container create 8284afb597804352ca99b0c35bfa8b12e2bf28fd0e4e05eccbafb1f1331d728c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:30:13.603 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 podman[99500]: 2024-04-27 02:30:13.260279439 +0000 UTC m=+0.431239633 container init 8284afb597804352ca99b0c35bfa8b12e2bf28fd0e4e05eccbafb1f1331d728c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-27T02:30:13.603 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 podman[99500]: 2024-04-27 02:30:13.263703635 +0000 UTC m=+0.434663827 container start 8284afb597804352ca99b0c35bfa8b12e2bf28fd0e4e05eccbafb1f1331d728c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:30:13.603 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 podman[99500]: 2024-04-27 02:30:13.35092686 +0000 UTC m=+0.521887071 container attach 8284afb597804352ca99b0c35bfa8b12e2bf28fd0e4e05eccbafb1f1331d728c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-27T02:30:13.890 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:30:13.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 94/723 objects degraded (13.001%), 18 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:13.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 94/723 objects degraded (13.001%), 18 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:13.988 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate[99515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-27T02:30:13.988 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate[99515]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-27T02:30:13.988 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 bash[99500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-27T02:30:13.988 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 bash[99500]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-27T02:30:13.988 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 bash[99500]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 bash[99500]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 bash[99500]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 bash[99500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 bash[99500]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate[99515]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate[99515]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate[99515]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate[99515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate[99515]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-27T02:30:13.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:13 smithi026 podman[99500]: 2024-04-27 02:30:13.752138695 +0000 UTC m=+0.923098881 container died 8284afb597804352ca99b0c35bfa8b12e2bf28fd0e4e05eccbafb1f1331d728c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-27T02:30:14.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:13 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 94/723 objects degraded (13.001%), 18 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:14.353 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 podman[99659]: 2024-04-27 02:30:14.26336776 +0000 UTC m=+0.504347895 container remove 8284afb597804352ca99b0c35bfa8b12e2bf28fd0e4e05eccbafb1f1331d728c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-27T02:30:14.589 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:30:14.589 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (4m) 15s ago 26m 16.4M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:30:14.589 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (3m) 104s ago 25m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:30:14.589 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (4m) 15s ago 25m 59.8M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:30:14.589 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (5m) 104s ago 30m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (15m) 15s ago 32m 501M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (117s) 15s ago 32m 43.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (2m) 104s ago 30m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (93s) 15s ago 31m 37.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 15s ago 26m 8010k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (4m) 104s ago 26m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (20s) 15s ago 30m 11.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (29m) 15s ago 29m 55.6M 2993M 17.2.0 e1d6a67b021e 8005748a9388 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (29m) 15s ago 29m 51.4M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (29m) 15s ago 29m 57.7M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (28m) 104s ago 28m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (28m) 104s ago 28m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (28m) 104s ago 28m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (27m) 104s ago 27m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:30:14.590 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (4m) 104s ago 26m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:30:14.591 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (25m) 15s ago 25m 77.8M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:30:14.591 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (25m) 104s ago 25m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:30:14.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:14 smithi026 ceph-mon[87214]: pgmap v64: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:14.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:14 smithi026 ceph-mon[87214]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:14.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:14 smithi026 ceph-mon[87214]: from='client.44256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:14.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:14 smithi026 ceph-mon[82260]: pgmap v64: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:14.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:14 smithi026 ceph-mon[82260]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:14.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:14 smithi026 ceph-mon[82260]: from='client.44256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:14.717 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 podman[99753]: 2024-04-27 02:30:14.38496175 +0000 UTC m=+0.019444381 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:30:14.717 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 podman[99753]: 2024-04-27 02:30:14.48956488 +0000 UTC m=+0.124047491 container create 00c20346f672b996481ad11d9ca15d59beda99f964d9a3e7d25883342e4cd9f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0) 2024-04-27T02:30:14.718 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 podman[99753]: 2024-04-27 02:30:14.716736153 +0000 UTC m=+0.351218765 container init 00c20346f672b996481ad11d9ca15d59beda99f964d9a3e7d25883342e4cd9f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5) 2024-04-27T02:30:14.718 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 podman[99753]: 2024-04-27 02:30:14.720048506 +0000 UTC m=+0.354531118 container start 00c20346f672b996481ad11d9ca15d59beda99f964d9a3e7d25883342e4cd9f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-27T02:30:15.016 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:30:15.016 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:30:15.016 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-27T02:30:15.017 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 6 2024-04-27T02:30:15.018 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:30:15.018 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:30:15.018 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 ceph-osd[99796]: -- 172.21.15.26:0/4016575898 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f8cda19680 con 0x55f8cd9e2c00 2024-04-27T02:30:15.018 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 bash[99753]: 00c20346f672b996481ad11d9ca15d59beda99f964d9a3e7d25883342e4cd9f1 2024-04-27T02:30:15.018 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:14 smithi026 systemd[1]: Started Ceph osd.1 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:30:15.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:14 smithi151 ceph-mon[64897]: pgmap v64: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:15.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:14 smithi151 ceph-mon[64897]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:15.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:14 smithi151 ceph-mon[64897]: from='client.44256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:15.353 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[99787]: 2024-04-27T02:30:15.267+0000 7f607b5507c0 -1 Falling back to public interface 2024-04-27T02:30:15.391 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (0 remaining).", 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "1/8 daemons upgraded", 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:30:15.392 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:30:15.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[82260]: from='client.44262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/1974951326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[82260]: from='client.44277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[87214]: from='client.44262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:15.979 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:15.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:15.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/1974951326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:15.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:15 smithi026 ceph-mon[87214]: from='client.44277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:16.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:15 smithi151 ceph-mon[64897]: from='client.44262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:16.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:16.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:16.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:16.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:15 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/1974951326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:16.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:15 smithi151 ceph-mon[64897]: from='client.44277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:16.974 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:16 smithi026 ceph-mon[82260]: pgmap v65: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:16.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:16 smithi026 ceph-mon[87214]: pgmap v65: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:17.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:16 smithi151 ceph-mon[64897]: pgmap v65: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:18.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:17 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:18.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:17 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:18.121 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:17 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[99787]: 2024-04-27T02:30:17.995+0000 7f607b5507c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-27T02:30:18.121 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[99787]: 2024-04-27T02:30:18.028+0000 7f607b5507c0 -1 osd.1 116 log_to_monitors true 2024-04-27T02:30:18.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:17 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:19.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:18 smithi151 ceph-mon[64897]: pgmap v66: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:19.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:18 smithi151 ceph-mon[64897]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-27T02:30:19.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:18 smithi026 ceph-mon[82260]: pgmap v66: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:19.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:18 smithi026 ceph-mon[82260]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-27T02:30:19.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:18 smithi026 ceph-mon[87214]: pgmap v66: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 121 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:19.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:18 smithi026 ceph-mon[87214]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-27T02:30:19.353 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:30:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[99787]: 2024-04-27T02:30:18.925+0000 7f606c9a8700 -1 osd.1 116 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:30:20.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:19 smithi151 ceph-mon[64897]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-27T02:30:20.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:19 smithi151 ceph-mon[64897]: osdmap e119: 8 total, 7 up, 8 in 2024-04-27T02:30:20.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:19 smithi151 ceph-mon[64897]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:30:20.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:19 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:20.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:19 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[82260]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[82260]: osdmap e119: 8 total, 7 up, 8 in 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[82260]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[87214]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[87214]: osdmap e119: 8 total, 7 up, 8 in 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[87214]: from='osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:20.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:19 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:21.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[87214]: pgmap v68: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:21.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:30:21.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[87214]: osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706] boot 2024-04-27T02:30:21.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[87214]: osdmap e120: 8 total, 8 up, 8 in 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[82260]: pgmap v68: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[82260]: osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706] boot 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[82260]: osdmap e120: 8 total, 8 up, 8 in 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:21.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:20 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:21.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:20 smithi151 ceph-mon[64897]: pgmap v68: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-04-27T02:30:21.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:20 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:30:21.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:20 smithi151 ceph-mon[64897]: osd.1 [v2:172.21.15.26:6810/2614424706,v1:172.21.15.26:6811/2614424706] boot 2024-04-27T02:30:21.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:20 smithi151 ceph-mon[64897]: osdmap e120: 8 total, 8 up, 8 in 2024-04-27T02:30:21.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:20 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-27T02:30:21.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:20 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:21.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:20 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:22.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:21 smithi026 ceph-mon[82260]: osdmap e121: 8 total, 8 up, 8 in 2024-04-27T02:30:22.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:21 smithi026 ceph-mon[87214]: osdmap e121: 8 total, 8 up, 8 in 2024-04-27T02:30:22.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:21 smithi151 ceph-mon[64897]: osdmap e121: 8 total, 8 up, 8 in 2024-04-27T02:30:22.369 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:30:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:30:22] "GET /metrics HTTP/1.1" 200 38159 "" "Prometheus/2.43.0" 2024-04-27T02:30:23.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:22 smithi026 ceph-mon[82260]: pgmap v71: 161 pgs: 6 active+undersized+degraded, 20 active+undersized, 135 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 42/723 objects degraded (5.809%) 2024-04-27T02:30:23.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:22 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 42/723 objects degraded (5.809%), 6 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:23.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:22 smithi026 ceph-mon[87214]: pgmap v71: 161 pgs: 6 active+undersized+degraded, 20 active+undersized, 135 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 42/723 objects degraded (5.809%) 2024-04-27T02:30:23.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:22 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 42/723 objects degraded (5.809%), 6 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:23.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:22 smithi151 ceph-mon[64897]: pgmap v71: 161 pgs: 6 active+undersized+degraded, 20 active+undersized, 135 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 42/723 objects degraded (5.809%) 2024-04-27T02:30:23.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:22 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 42/723 objects degraded (5.809%), 6 pgs degraded (PG_DEGRADED) 2024-04-27T02:30:25.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: pgmap v72: 161 pgs: 5 active+undersized+degraded, 15 active+undersized, 141 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 36/723 objects degraded (4.979%) 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: Upgrade: Finalizing container_image settings 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:30:25.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:30:25.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:30:25.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:30:25.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:30:25.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:30:25.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:30:25.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: Upgrade: Complete! 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: pgmap v72: 161 pgs: 5 active+undersized+degraded, 15 active+undersized, 141 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 36/723 objects degraded (4.979%) 2024-04-27T02:30:25.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: Upgrade: Finalizing container_image settings 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:30:25.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:30:25.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:30:25.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:30:25.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:30:25.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: pgmap v72: 161 pgs: 5 active+undersized+degraded, 15 active+undersized, 141 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 36/723 objects degraded (4.979%) 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: Upgrade: Finalizing container_image settings 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:30:25.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: Upgrade: Complete! 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: Upgrade: Complete! 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:25.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:30:25.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:30:25.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:30:25.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:27.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:26 smithi151 ceph-mon[64897]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 742 B/s rd, 0 op/s 2024-04-27T02:30:27.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:27.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:27.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:27.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:26 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/723 objects degraded (4.979%), 5 pgs degraded) 2024-04-27T02:30:27.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:26 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[82260]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 742 B/s rd, 0 op/s 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/723 objects degraded (4.979%), 5 pgs degraded) 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[87214]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 742 B/s rd, 0 op/s 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:27.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/723 objects degraded (4.979%), 5 pgs degraded) 2024-04-27T02:30:27.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:26 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:30:28.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:27 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:28.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:27 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:28.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:27 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:29.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:28 smithi151 ceph-mon[64897]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:29.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:28 smithi026 ceph-mon[82260]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:29.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:28 smithi026 ceph-mon[87214]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:31.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:30 smithi151 ceph-mon[64897]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:30:31.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:30 smithi026 ceph-mon[82260]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:30:31.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:30 smithi026 ceph-mon[87214]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:30:32.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:32 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:32.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:32 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:32.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:32 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:32.604 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:30:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:30:32] "GET /metrics HTTP/1.1" 200 38160 "" "Prometheus/2.43.0" 2024-04-27T02:30:33.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:33 smithi151 ceph-mon[64897]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:30:33.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:33 smithi026 ceph-mon[82260]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:30:33.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:33 smithi026 ceph-mon[87214]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:30:35.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:35 smithi151 ceph-mon[64897]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:35.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:35 smithi026 ceph-mon[82260]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:35.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:35 smithi026 ceph-mon[87214]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:37.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:37 smithi151 ceph-mon[64897]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:37.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:37 smithi026 ceph-mon[82260]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:37.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:37 smithi026 ceph-mon[87214]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:38.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:38 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:38.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:38 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:38 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:39.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:39 smithi151 ceph-mon[64897]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:39 smithi026 ceph-mon[82260]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:39 smithi026 ceph-mon[87214]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:41 smithi151 ceph-mon[64897]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:41.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:41 smithi026 ceph-mon[82260]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:41.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:41.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:41 smithi026 ceph-mon[87214]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:30:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:42.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:30:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:30:42] "GET /metrics HTTP/1.1" 200 38174 "" "Prometheus/2.43.0" 2024-04-27T02:30:43.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:43 smithi151 ceph-mon[64897]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:43.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:43 smithi026 ceph-mon[82260]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:43.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:43 smithi026 ceph-mon[87214]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:45 smithi151 ceph-mon[64897]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:45.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:45 smithi026 ceph-mon[82260]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:45.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:45 smithi026 ceph-mon[87214]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:46.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:30:47.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:47 smithi026 ceph-mon[82260]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:47.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:47 smithi026 ceph-mon[82260]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:47.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:47 smithi026 ceph-mon[87214]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:47.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:47 smithi026 ceph-mon[87214]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:47.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:47 smithi151 ceph-mon[64897]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:47.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:47 smithi151 ceph-mon[64897]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:48.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:48.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:48.463 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:30:48.463 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (4m) 29s ago 26m 16.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:30:48.463 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (4m) 2m ago 26m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:30:48.463 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (5m) 29s ago 25m 59.8M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:30:48.463 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (5m) 2m ago 31m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:30:48.463 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (15m) 29s ago 32m 502M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:30:48.463 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (2m) 29s ago 33m 45.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (3m) 2m ago 31m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (2m) 29s ago 31m 37.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (5m) 29s ago 27m 7814k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (5m) 2m ago 27m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (54s) 29s ago 30m 48.5M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (34s) 29s ago 30m 15.1M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (30m) 29s ago 30m 52.5M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (29m) 29s ago 29m 58.2M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (29m) 2m ago 29m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (29m) 2m ago 28m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (28m) 2m ago 28m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (28m) 2m ago 28m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (4m) 2m ago 27m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:30:48.464 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (25m) 29s ago 25m 77.9M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:30:48.465 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (25m) 2m ago 25m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:30:48.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:49.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-27T02:30:49.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:49 smithi026 ceph-mon[82260]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:49.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:49 smithi026 ceph-mon[82260]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:49.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:49 smithi026 ceph-mon[87214]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:49.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:49 smithi026 ceph-mon[87214]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:49.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:49 smithi151 ceph-mon[64897]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:49.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:49 smithi151 ceph-mon[64897]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:51.172 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:30:51.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:51 smithi026 ceph-mon[82260]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:51.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:51 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/3363231836' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:51 smithi026 ceph-mon[87214]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:51 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/3363231836' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:51.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:51 smithi151 ceph-mon[64897]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:51.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:51 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/3363231836' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:30:51.798 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-04-27T02:30:52.389 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:30:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:30:52] "GET /metrics HTTP/1.1" 200 38174 "" "Prometheus/2.43.0" 2024-04-27T02:30:53.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:53 smithi026 ceph-mon[87214]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:53.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:53 smithi026 ceph-mon[82260]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:53.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:53 smithi151 ceph-mon[64897]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:55 smithi151 ceph-mon[64897]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:55 smithi151 ceph-mon[64897]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:55 smithi026 ceph-mon[82260]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:55 smithi026 ceph-mon[82260]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:55.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:55 smithi026 ceph-mon[87214]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:55.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:55 smithi026 ceph-mon[87214]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:55.834 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:30:56.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:56.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:56.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:30:56.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:30:57.483 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:57 smithi026 ceph-mon[82260]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:57.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:57 smithi026 ceph-mon[87214]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:57.495 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:57 smithi151 ceph-mon[64897]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:30:58.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:58 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:58.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:58 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:30:58.532 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:30:58.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:58 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:30:59.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:30:59.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:59 smithi026 ceph-mon[82260]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:59.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:30:59 smithi026 ceph-mon[82260]: from='client.44310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:59.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:59 smithi026 ceph-mon[87214]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:59.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:30:59 smithi026 ceph-mon[87214]: from='client.44310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:30:59.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:59 smithi151 ceph-mon[64897]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:30:59.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:30:59 smithi151 ceph-mon[64897]: from='client.44310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:01.224 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:31:01.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:01 smithi026 ceph-mon[82260]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:01.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:01 smithi026 ceph-mon[87214]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:01.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:01 smithi151 ceph-mon[64897]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:01.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-04-27T02:31:02.494 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:02 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/4250095532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:31:02.494 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:02 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/4250095532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:31:02.494 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:31:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:31:02] "GET /metrics HTTP/1.1" 200 38178 "" "Prometheus/2.43.0" 2024-04-27T02:31:02.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:02 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/4250095532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:31:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:03 smithi026 ceph-mon[82260]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:03.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:03 smithi026 ceph-mon[87214]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:03.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:03 smithi151 ceph-mon[64897]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:05.314 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:05 smithi026 ceph-mon[82260]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:05.314 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:05 smithi026 ceph-mon[82260]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:05.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:05 smithi026 ceph-mon[87214]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:05.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:05 smithi026 ceph-mon[87214]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:05.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:05 smithi151 ceph-mon[64897]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:05.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:05 smithi151 ceph-mon[64897]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:06.231 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:31:06.813 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-27T02:31:07.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[87214]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:07.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[87214]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:31:07.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:07.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:07.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[82260]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[82260]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:07.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:07.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:07 smithi151 ceph-mon[64897]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:07.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:07 smithi151 ceph-mon[64897]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:31:07.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:07.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:07.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:07.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:07.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:08.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:08 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:08.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:08 smithi026 ceph-mon[82260]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:31:08.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:08 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:08.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:08 smithi026 ceph-mon[87214]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:31:08.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:08 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:08.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:08 smithi151 ceph-mon[64897]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:31:09.050 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:31:09.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:09 smithi026 ceph-mon[82260]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:09.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:09 smithi026 ceph-mon[87214]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:09 smithi151 ceph-mon[64897]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 50s ago 26m 16.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (4m) 2m ago 26m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (5m) 50s ago 26m 59.8M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (6m) 2m ago 31m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (16m) 50s ago 33m 502M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (2m) 50s ago 33m 45.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:31:09.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (3m) 2m ago 31m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (2m) 50s ago 31m 37.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (5m) 50s ago 27m 7814k - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (5m) 2m ago 27m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (75s) 50s ago 31m 48.5M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (55s) 50s ago 30m 15.1M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (30m) 50s ago 30m 52.5M 2993M 17.2.0 e1d6a67b021e ad81062c75aa 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (30m) 50s ago 30m 58.2M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (29m) 2m ago 29m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (29m) 2m ago 29m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (28m) 2m ago 28m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (28m) 2m ago 28m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (5m) 2m ago 27m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:31:09.737 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (26m) 50s ago 26m 77.9M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:31:09.738 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (26m) 2m ago 26m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:31:10.161 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:31:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:31:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:31:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:31:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:31:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 7 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:31:10.162 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='client.44328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2762727199' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='client.44328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-27T02:31:10.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:10 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2762727199' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.530 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:31:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:31:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:31:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-04-27T02:31:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:31:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/8 daemons upgraded", 2024-04-27T02:31:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:31:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:31:10.531 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:31:10.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='client.44328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:10.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:10.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:10.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:10.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:10.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-27T02:31:10.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:10 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2762727199' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:11.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:11.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:11.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mgr 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mon 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all crash 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: Upgrade: osd.2 is safe to restart 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:11.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mgr 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mon 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all crash 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: Upgrade: osd.2 is safe to restart 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:11.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mgr 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mon 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all crash 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: Upgrade: osd.2 is safe to restart 2024-04-27T02:31:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:11.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:12.452 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:31:12] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-04-27T02:31:12.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[82260]: Upgrade: Updating osd.2 2024-04-27T02:31:12.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:12.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:31:12.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:12.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[82260]: Deploying daemon osd.2 on smithi026 2024-04-27T02:31:12.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[87214]: Upgrade: Updating osd.2 2024-04-27T02:31:12.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:12.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:31:12.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:12.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:12 smithi026 ceph-mon[87214]: Deploying daemon osd.2 on smithi026 2024-04-27T02:31:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:12 smithi151 ceph-mon[64897]: Upgrade: Updating osd.2 2024-04-27T02:31:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:12 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:12 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-27T02:31:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:12 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:12.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:12 smithi151 ceph-mon[64897]: Deploying daemon osd.2 on smithi026 2024-04-27T02:31:13.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:13 smithi151 ceph-mon[64897]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:13.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:13 smithi026 ceph-mon[82260]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:13.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:13 smithi026 ceph-mon[87214]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:15.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:14 smithi026 systemd[1]: Stopping Ceph osd.2 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:31:15.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[35340]: 2024-04-27T02:31:15.144+0000 7f695fbd8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:31:15.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[35340]: 2024-04-27T02:31:15.144+0000 7f695fbd8700 -1 osd.2 121 *** Got signal Terminated *** 2024-04-27T02:31:15.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[35340]: 2024-04-27T02:31:15.144+0000 7f695fbd8700 -1 osd.2 121 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:31:15.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:15 smithi151 ceph-mon[64897]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:15.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:15 smithi151 ceph-mon[64897]: osd.2 marked itself down and dead 2024-04-27T02:31:15.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:15 smithi026 ceph-mon[82260]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:15.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:15 smithi026 ceph-mon[82260]: osd.2 marked itself down and dead 2024-04-27T02:31:15.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:15 smithi026 ceph-mon[87214]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:15.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:15 smithi026 ceph-mon[87214]: osd.2 marked itself down and dead 2024-04-27T02:31:16.277 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:15 smithi026 podman[106024]: 2024-04-27 02:31:15.956032966 +0000 UTC m=+0.895680781 container died ad81062c75aa38f35e703a90e5789f03eadbe5dbf148f80f042302878132e9ac (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-27T02:31:16.530 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:16 smithi026 podman[106024]: 2024-04-27 02:31:16.303189553 +0000 UTC m=+1.242837369 container cleanup ad81062c75aa38f35e703a90e5789f03eadbe5dbf148f80f042302878132e9ac (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, ceph=True, GIT_CLEAN=True, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, architecture=x86_64, name=centos-stream) 2024-04-27T02:31:16.531 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:16 smithi026 bash[106024]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2 2024-04-27T02:31:16.531 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:16 smithi026 podman[106043]: 2024-04-27 02:31:16.496424448 +0000 UTC m=+0.537775778 container remove ad81062c75aa38f35e703a90e5789f03eadbe5dbf148f80f042302878132e9ac (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , ceph=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, release=754, vcs-type=git, version=8) 2024-04-27T02:31:16.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:16 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:31:16.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:16 smithi151 ceph-mon[64897]: osdmap e122: 8 total, 7 up, 8 in 2024-04-27T02:31:16.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:16 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:31:16.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:16 smithi026 ceph-mon[82260]: osdmap e122: 8 total, 7 up, 8 in 2024-04-27T02:31:16.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:16 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:31:16.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:16 smithi026 ceph-mon[87214]: osdmap e122: 8 total, 7 up, 8 in 2024-04-27T02:31:17.224 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:16 smithi026 podman[106137]: 2024-04-27 02:31:16.850245919 +0000 UTC m=+0.018280708 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:31:17.224 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:17 smithi026 podman[106137]: 2024-04-27 02:31:17.0057442 +0000 UTC m=+0.173778979 container create 503e421a600ed7fae97e95ea6d3b73d744f2783ff9cac859d42a930f9d1c1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-27T02:31:17.539 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:17 smithi026 podman[106137]: 2024-04-27 02:31:17.25017604 +0000 UTC m=+0.418210822 container init 503e421a600ed7fae97e95ea6d3b73d744f2783ff9cac859d42a930f9d1c1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-27T02:31:17.539 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:17 smithi026 podman[106137]: 2024-04-27 02:31:17.25351686 +0000 UTC m=+0.421551640 container start 503e421a600ed7fae97e95ea6d3b73d744f2783ff9cac859d42a930f9d1c1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:31:17.539 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:17 smithi026 podman[106137]: 2024-04-27 02:31:17.341635274 +0000 UTC m=+0.509670061 container attach 503e421a600ed7fae97e95ea6d3b73d744f2783ff9cac859d42a930f9d1c1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-27T02:31:17.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:17 smithi151 ceph-mon[64897]: pgmap v99: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:31:17.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:17 smithi151 ceph-mon[64897]: osdmap e123: 8 total, 7 up, 8 in 2024-04-27T02:31:17.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:17 smithi026 ceph-mon[82260]: pgmap v99: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:31:17.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:17 smithi026 ceph-mon[82260]: osdmap e123: 8 total, 7 up, 8 in 2024-04-27T02:31:17.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:17 smithi026 ceph-mon[87214]: pgmap v99: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:31:17.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:17 smithi026 ceph-mon[87214]: osdmap e123: 8 total, 7 up, 8 in 2024-04-27T02:31:17.853 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:17 smithi026 podman[106137]: 2024-04-27 02:31:17.566351732 +0000 UTC m=+0.734386517 container died 503e421a600ed7fae97e95ea6d3b73d744f2783ff9cac859d42a930f9d1c1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-27T02:31:18.544 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 podman[106163]: 2024-04-27 02:31:18.19595145 +0000 UTC m=+0.623684438 container remove 503e421a600ed7fae97e95ea6d3b73d744f2783ff9cac859d42a930f9d1c1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-27T02:31:18.544 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-27T02:31:18.544 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 systemd[1]: Stopped Ceph osd.2 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:31:18.544 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.2.service: Consumed 13.424s CPU time. 2024-04-27T02:31:18.545 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 systemd[1]: Starting Ceph osd.2 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:31:18.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:18 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:18.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:18 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:18.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:18 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:18.853 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 podman[106269]: 2024-04-27 02:31:18.562112124 +0000 UTC m=+0.018625242 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:31:18.853 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 podman[106269]: 2024-04-27 02:31:18.690071178 +0000 UTC m=+0.146584318 container create 144b43493d3c45748e42e7a776152a0d9d6a764c11fb29222bc5501a2797dfe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-27T02:31:19.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 podman[106269]: 2024-04-27 02:31:18.933131513 +0000 UTC m=+0.389644611 container init 144b43493d3c45748e42e7a776152a0d9d6a764c11fb29222bc5501a2797dfe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-27T02:31:19.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:18 smithi026 podman[106269]: 2024-04-27 02:31:18.936478142 +0000 UTC m=+0.392991242 container start 144b43493d3c45748e42e7a776152a0d9d6a764c11fb29222bc5501a2797dfe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-27T02:31:19.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 podman[106269]: 2024-04-27 02:31:19.033273999 +0000 UTC m=+0.489787116 container attach 144b43493d3c45748e42e7a776152a0d9d6a764c11fb29222bc5501a2797dfe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-27T02:31:19.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-mon[82260]: pgmap v101: 161 pgs: 12 active+undersized+degraded, 4 stale+active+clean, 22 active+undersized, 123 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 41/723 objects degraded (5.671%) 2024-04-27T02:31:19.695 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 12 pgs degraded (PG_DEGRADED) 2024-04-27T02:31:19.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-mon[87214]: pgmap v101: 161 pgs: 12 active+undersized+degraded, 4 stale+active+clean, 22 active+undersized, 123 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 41/723 objects degraded (5.671%) 2024-04-27T02:31:19.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 12 pgs degraded (PG_DEGRADED) 2024-04-27T02:31:19.695 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate[106285]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-27T02:31:19.695 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate[106285]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-27T02:31:19.695 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 bash[106269]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-27T02:31:19.695 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 bash[106269]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 bash[106269]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 bash[106269]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 bash[106269]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 bash[106269]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 bash[106269]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate[106285]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate[106285]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate[106285]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate[106285]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate[106285]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-27T02:31:19.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:19 smithi026 podman[106269]: 2024-04-27 02:31:19.420169416 +0000 UTC m=+0.876682532 container died 144b43493d3c45748e42e7a776152a0d9d6a764c11fb29222bc5501a2797dfe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb) 2024-04-27T02:31:19.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:19 smithi151 ceph-mon[64897]: pgmap v101: 161 pgs: 12 active+undersized+degraded, 4 stale+active+clean, 22 active+undersized, 123 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 41/723 objects degraded (5.671%) 2024-04-27T02:31:19.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:19 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 12 pgs degraded (PG_DEGRADED) 2024-04-27T02:31:20.262 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 podman[106415]: 2024-04-27 02:31:20.029165495 +0000 UTC m=+0.603244600 container remove 144b43493d3c45748e42e7a776152a0d9d6a764c11fb29222bc5501a2797dfe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-04-27T02:31:20.262 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 podman[106481]: 2024-04-27 02:31:20.153675918 +0000 UTC m=+0.019097615 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:31:20.263 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 podman[106481]: 2024-04-27 02:31:20.28862757 +0000 UTC m=+0.154049250 container create 77827e7c5612a78a293678c4d574b7ef7838f577e139d9c6dbfc17f7f6f5f809 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:31:20.515 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 podman[106481]: 2024-04-27 02:31:20.516222508 +0000 UTC m=+0.381644189 container init 77827e7c5612a78a293678c4d574b7ef7838f577e139d9c6dbfc17f7f6f5f809 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-27T02:31:20.515 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 podman[106481]: 2024-04-27 02:31:20.519606403 +0000 UTC m=+0.385028089 container start 77827e7c5612a78a293678c4d574b7ef7838f577e139d9c6dbfc17f7f6f5f809 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:31:20.853 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 ceph-osd[106500]: -- 172.21.15.26:0/1903663105 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55928a063680 con 0x55928a02d000 2024-04-27T02:31:20.853 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 bash[106481]: 77827e7c5612a78a293678c4d574b7ef7838f577e139d9c6dbfc17f7f6f5f809 2024-04-27T02:31:20.853 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:20 smithi026 systemd[1]: Started Ceph osd.2 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:31:21.353 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[106495]: 2024-04-27T02:31:21.060+0000 7fd6248377c0 -1 Falling back to public interface 2024-04-27T02:31:21.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:21 smithi151 ceph-mon[64897]: pgmap v102: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:21.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:21 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:21 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:21 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:21 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:21.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[87214]: pgmap v102: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:21.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:21.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[82260]: pgmap v102: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:21.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:21.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:21 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:22.470 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:31:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:31:22] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-04-27T02:31:23.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:23 smithi026 ceph-mon[82260]: pgmap v103: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:23.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:23 smithi026 ceph-mon[87214]: pgmap v103: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:23.805 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[106495]: 2024-04-27T02:31:23.783+0000 7fd6248377c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-27T02:31:23.806 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[106495]: 2024-04-27T02:31:23.805+0000 7fd6248377c0 -1 osd.2 121 log_to_monitors true 2024-04-27T02:31:23.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:23 smithi151 ceph-mon[64897]: pgmap v103: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:24.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:24 smithi151 ceph-mon[64897]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:31:24.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:24 smithi026 ceph-mon[87214]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:31:24.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:24 smithi026 ceph-mon[82260]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-27T02:31:24.853 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:31:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[106495]: 2024-04-27T02:31:24.601+0000 7fd615c8f700 -1 osd.2 121 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:31:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:25 smithi151 ceph-mon[64897]: pgmap v104: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:25 smithi151 ceph-mon[64897]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-27T02:31:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:25 smithi151 ceph-mon[64897]: osdmap e124: 8 total, 7 up, 8 in 2024-04-27T02:31:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:25 smithi151 ceph-mon[64897]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:31:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:25.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[82260]: pgmap v104: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[82260]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[82260]: osdmap e124: 8 total, 7 up, 8 in 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[82260]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[87214]: pgmap v104: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[87214]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[87214]: osdmap e124: 8 total, 7 up, 8 in 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[87214]: from='osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:25.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291] boot 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: osdmap e125: 8 total, 8 up, 8 in 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:26 smithi151 ceph-mon[64897]: osdmap e126: 8 total, 8 up, 8 in 2024-04-27T02:31:26.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291] boot 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: osdmap e125: 8 total, 8 up, 8 in 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[82260]: osdmap e126: 8 total, 8 up, 8 in 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: osd.2 [v2:172.21.15.26:6818/1470922291,v1:172.21.15.26:6819/1470922291] boot 2024-04-27T02:31:26.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: osdmap e125: 8 total, 8 up, 8 in 2024-04-27T02:31:26.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-27T02:31:26.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:26.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:26.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:26 smithi026 ceph-mon[87214]: osdmap e126: 8 total, 8 up, 8 in 2024-04-27T02:31:27.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:27 smithi151 ceph-mon[64897]: pgmap v107: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:27.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:27 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 15 pgs degraded (PG_DEGRADED) 2024-04-27T02:31:27.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:27 smithi026 ceph-mon[82260]: pgmap v107: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:27.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:27 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 15 pgs degraded (PG_DEGRADED) 2024-04-27T02:31:27.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:27 smithi026 ceph-mon[87214]: pgmap v107: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:31:27.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:27 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 15 pgs degraded (PG_DEGRADED) 2024-04-27T02:31:28.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:28 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:28.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:28 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:28.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:28 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:29.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:29 smithi026 ceph-mon[82260]: pgmap v109: 161 pgs: 4 active+undersized+degraded, 17 active+undersized, 140 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 30/723 objects degraded (4.149%) 2024-04-27T02:31:29.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:29 smithi026 ceph-mon[87214]: pgmap v109: 161 pgs: 4 active+undersized+degraded, 17 active+undersized, 140 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 30/723 objects degraded (4.149%) 2024-04-27T02:31:29.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:29 smithi151 ceph-mon[64897]: pgmap v109: 161 pgs: 4 active+undersized+degraded, 17 active+undersized, 140 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 30/723 objects degraded (4.149%) 2024-04-27T02:31:31.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:31.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: Upgrade: Finalizing container_image settings 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: Upgrade: Complete! 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.347 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/723 objects degraded (4.149%), 4 pgs degraded) 2024-04-27T02:31:31.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:30 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:31:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: Upgrade: Finalizing container_image settings 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:31:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:31:31.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:31:31.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:31:31.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:31:31.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:31:31.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:31:31.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: Upgrade: Finalizing container_image settings 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:31:31.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: Upgrade: Complete! 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/723 objects degraded (4.149%), 4 pgs degraded) 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:31:31.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: Upgrade: Complete! 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:31:31.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:31.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/723 objects degraded (4.149%), 4 pgs degraded) 2024-04-27T02:31:31.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:30 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:31:32.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:32 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:32.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:32 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:32.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:32 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:32.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:31:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:31:32] "GET /metrics HTTP/1.1" 200 38038 "" "Prometheus/2.43.0" 2024-04-27T02:31:33.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:33 smithi151 ceph-mon[64897]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:31:33.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:33 smithi026 ceph-mon[82260]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:31:33.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:33 smithi026 ceph-mon[87214]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:31:35.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:35 smithi151 ceph-mon[64897]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:35.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:35 smithi026 ceph-mon[82260]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:35.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:35 smithi026 ceph-mon[87214]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:37.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:37 smithi026 ceph-mon[82260]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:31:37.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:37.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:37 smithi026 ceph-mon[87214]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:31:37.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:37.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:37 smithi151 ceph-mon[64897]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:31:37.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:38.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:38 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:38.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:38 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:38 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:39.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:39 smithi151 ceph-mon[64897]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:31:39.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:39 smithi026 ceph-mon[82260]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:31:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:39 smithi026 ceph-mon[87214]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:31:41.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:41 smithi026 ceph-mon[82260]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:41.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:41.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:41.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:41 smithi026 ceph-mon[87214]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:41.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:41.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:41.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:31:41.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:41 smithi151 ceph-mon[64897]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:31:41.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:42.529 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:42 smithi026 ceph-mon[82260]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:42.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:42 smithi026 ceph-mon[87214]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:42.529 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:31:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:31:42] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-04-27T02:31:42.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:42 smithi151 ceph-mon[64897]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:43.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:43 smithi026 ceph-mon[82260]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:43.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:43 smithi026 ceph-mon[87214]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 18s ago 27m 26.0M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (5m) 3m ago 27m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (6m) 18s ago 26m 62.0M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (6m) 3m ago 32m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (16m) 18s ago 33m 505M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) 18s ago 33m 48.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (4m) 3m ago 32m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 18s ago 32m 40.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:31:43.511 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 18s ago 28m 16.7M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (6m) 3m ago 28m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (109s) 18s ago 31m 50.3M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (89s) 18s ago 31m 46.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (23s) 18s ago 31m 12.4M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (30m) 18s ago 30m 59.8M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (30m) 3m ago 30m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (29m) 3m ago 29m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (29m) 3m ago 29m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (29m) 3m ago 29m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (5m) 3m ago 28m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (26m) 18s ago 26m 78.2M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:31:43.512 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (26m) 3m ago 26m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:31:43.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:43 smithi151 ceph-mon[64897]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:44.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-27T02:31:44.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:44 smithi026 ceph-mon[82260]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:44.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:44 smithi026 ceph-mon[87214]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:44 smithi151 ceph-mon[64897]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:45.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:45 smithi026 ceph-mon[82260]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:45.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:45 smithi026 ceph-mon[87214]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:45.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:45 smithi151 ceph-mon[64897]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:46.220 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:31:46.795 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-04-27T02:31:47.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:47 smithi026 ceph-mon[82260]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:47.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:47 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2529572206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:47.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:47 smithi026 ceph-mon[87214]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:47.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:47 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2529572206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:47.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:47 smithi151 ceph-mon[64897]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:47.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:47 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2529572206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:31:48.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:48.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:48.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:49.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:49 smithi151 ceph-mon[64897]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:49.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:49 smithi026 ceph-mon[82260]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:49.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:49 smithi026 ceph-mon[87214]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:50.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:50 smithi026 ceph-mon[82260]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:50.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:50 smithi026 ceph-mon[87214]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:50.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:50 smithi151 ceph-mon[64897]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:50.917 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:31:51.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:51 smithi026 ceph-mon[82260]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:51.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:51 smithi026 ceph-mon[87214]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:51.534 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:31:51.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:51 smithi151 ceph-mon[64897]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:52.356 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:31:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:31:52] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-04-27T02:31:53.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:53 smithi151 ceph-mon[64897]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:53.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:53 smithi026 ceph-mon[82260]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:53.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:53 smithi026 ceph-mon[87214]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:53.641 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:31:53.641 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:31:53.641 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:31:53.641 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:31:53.641 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:31:53.641 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:31:53.642 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:31:53.642 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:31:53.642 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:31:54.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:31:55.316 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:55 smithi026 ceph-mon[82260]: from='client.54325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:55.316 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:55 smithi026 ceph-mon[82260]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:55.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:55 smithi026 ceph-mon[87214]: from='client.54325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:55.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:55 smithi026 ceph-mon[87214]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:55.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:55 smithi151 ceph-mon[64897]: from='client.54325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:31:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:55 smithi151 ceph-mon[64897]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:56.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:56.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:56.511 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:31:56.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:31:57.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-04-27T02:31:57.494 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:57 smithi151 ceph-mon[64897]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:57.495 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:57 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/762847479' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:31:57.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:57 smithi026 ceph-mon[82260]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:57.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:57 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/762847479' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:31:57.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:57 smithi026 ceph-mon[87214]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:31:57.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:57 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/762847479' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:31:58.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:58 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:58.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:58 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:58.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:58 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:31:59.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:31:59 smithi151 ceph-mon[64897]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:59.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:31:59 smithi026 ceph-mon[82260]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:31:59.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:31:59 smithi026 ceph-mon[87214]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:00.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:00 smithi026 ceph-mon[82260]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:00.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:00 smithi026 ceph-mon[87214]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:00.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:00 smithi151 ceph-mon[64897]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:01.412 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:32:01.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:01 smithi151 ceph-mon[64897]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:01.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:01 smithi026 ceph-mon[82260]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:01.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:01 smithi026 ceph-mon[87214]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:02.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-27T02:32:02.352 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:32:02] "GET /metrics HTTP/1.1" 200 38182 "" "Prometheus/2.43.0" 2024-04-27T02:32:02.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[82260]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:32:02.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:02.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:02.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:02.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:32:02.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:02.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[82260]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:32:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[87214]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:32:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:32:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:02 smithi026 ceph-mon[87214]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:32:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:02 smithi151 ceph-mon[64897]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:32:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:02 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:02 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:02 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:02 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:32:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:02 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:02.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:02 smithi151 ceph-mon[64897]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:32:03.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:03 smithi026 ceph-mon[82260]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:03.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:03 smithi026 ceph-mon[87214]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:03.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:03 smithi151 ceph-mon[64897]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:04.502 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:32:05.205 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:32:05.205 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 40s ago 27m 26.0M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:32:05.205 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (5m) 3m ago 27m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:32:05.205 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (6m) 40s ago 27m 62.0M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (6m) 3m ago 32m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (17m) 40s ago 34m 505M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) 40s ago 34m 48.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (4m) 3m ago 32m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 40s ago 32m 40.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 40s ago 28m 16.7M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (6m) 3m ago 28m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) 40s ago 32m 50.3M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (110s) 40s ago 31m 46.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (45s) 40s ago 31m 12.4M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (31m) 40s ago 31m 59.8M 2993M 17.2.0 e1d6a67b021e 95c4b2e6044d 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (30m) 3m ago 30m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (30m) 3m ago 30m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:32:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (29m) 3m ago 29m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:32:05.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (29m) 3m ago 29m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:32:05.207 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (6m) 3m ago 28m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:32:05.207 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (27m) 40s ago 27m 78.2M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:32:05.207 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (27m) 3m ago 27m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:32:05.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:05.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:32:05.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:05.642 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:32:05.642 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-27T02:32:05.642 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 8 2024-04-27T02:32:05.642 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:32:05.642 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:32:05.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:05.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-27T02:32:05.995 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:32:05.995 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:32:05.995 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:32:05.995 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-27T02:32:05.995 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:32:05.995 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "3/8 daemons upgraded", 2024-04-27T02:32:05.996 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:32:05.996 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:32:05.996 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:32:06.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mgr 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mon 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all crash 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: Upgrade: osd.3 is safe to restart 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2615786822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:06.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mgr 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mon 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all crash 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: Upgrade: osd.3 is safe to restart 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:06.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:06 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2615786822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:06.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mgr 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mon 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all crash 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: Upgrade: osd.3 is safe to restart 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:06 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2615786822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:07.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[82260]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:07.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[82260]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:07.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:07.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:32:07.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:07.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[87214]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:07.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[87214]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:07.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:07.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:32:07.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:07 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:07.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:07 smithi151 ceph-mon[64897]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:07.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:07 smithi151 ceph-mon[64897]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:07.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:07.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-27T02:32:07.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:07 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:08.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:08 smithi026 ceph-mon[87214]: Upgrade: Updating osd.3 2024-04-27T02:32:08.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:08 smithi026 ceph-mon[87214]: Deploying daemon osd.3 on smithi026 2024-04-27T02:32:08.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:08 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:08.686 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:08 smithi026 ceph-mon[82260]: Upgrade: Updating osd.3 2024-04-27T02:32:08.686 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:08 smithi026 ceph-mon[82260]: Deploying daemon osd.3 on smithi026 2024-04-27T02:32:08.686 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:08 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:08.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:08 smithi151 ceph-mon[64897]: Upgrade: Updating osd.3 2024-04-27T02:32:08.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:08 smithi151 ceph-mon[64897]: Deploying daemon osd.3 on smithi026 2024-04-27T02:32:08.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:08 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:09 smithi026 ceph-mon[82260]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:09.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:09 smithi026 ceph-mon[87214]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:09.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:09 smithi151 ceph-mon[64897]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:10.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:10 smithi026 systemd[1]: Stopping Ceph osd.3 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:32:10.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:10 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[38337]: 2024-04-27T02:32:10.349+0000 7f3a25109700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:32:10.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:10 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[38337]: 2024-04-27T02:32:10.349+0000 7f3a25109700 -1 osd.3 126 *** Got signal Terminated *** 2024-04-27T02:32:10.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:10 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[38337]: 2024-04-27T02:32:10.349+0000 7f3a25109700 -1 osd.3 126 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:32:10.825 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:10 smithi026 ceph-mon[82260]: osd.3 marked itself down and dead 2024-04-27T02:32:10.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:10 smithi026 ceph-mon[87214]: osd.3 marked itself down and dead 2024-04-27T02:32:10.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:10 smithi151 ceph-mon[64897]: osd.3 marked itself down and dead 2024-04-27T02:32:11.102 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:10 smithi026 podman[112517]: 2024-04-27 02:32:10.851312902 +0000 UTC m=+0.599540888 container died 95c4b2e6044d0ddbc0a224fc14f32e6a0594c251918a6e884e9a326614d91a0b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, vcs-type=git, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, RELEASE=HEAD, io.openshift.expose-services=, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-27T02:32:11.453 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:11 smithi026 podman[112517]: 2024-04-27 02:32:11.341172867 +0000 UTC m=+1.089400850 container cleanup 95c4b2e6044d0ddbc0a224fc14f32e6a0594c251918a6e884e9a326614d91a0b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.component=centos-stream-container, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-27T02:32:11.453 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:11 smithi026 bash[112517]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3 2024-04-27T02:32:11.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:11 smithi151 ceph-mon[64897]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:11.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:11 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:32:11.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:11 smithi151 ceph-mon[64897]: osdmap e127: 8 total, 7 up, 8 in 2024-04-27T02:32:11.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:11.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:11.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[87214]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:11.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[87214]: osdmap e127: 8 total, 7 up, 8 in 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[82260]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[82260]: osdmap e127: 8 total, 7 up, 8 in 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:11.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:11.853 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:11 smithi026 podman[112534]: 2024-04-27 02:32:11.617920103 +0000 UTC m=+0.761176898 container remove 95c4b2e6044d0ddbc0a224fc14f32e6a0594c251918a6e884e9a326614d91a0b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, com.redhat.component=centos-stream-container, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, release=754, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream) 2024-04-27T02:32:12.197 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:11 smithi026 podman[112627]: 2024-04-27 02:32:11.871457759 +0000 UTC m=+0.018487135 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:32:12.197 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:12 smithi026 podman[112627]: 2024-04-27 02:32:12.119117548 +0000 UTC m=+0.266146910 container create 2956e555c0254b3e5301b48e9f38148958e6c1259557d363d40aa5cfd6d3fdc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:32:12.464 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:32:12] "GET /metrics HTTP/1.1" 200 38185 "" "Prometheus/2.43.0" 2024-04-27T02:32:12.464 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:12 smithi026 podman[112627]: 2024-04-27 02:32:12.322272105 +0000 UTC m=+0.469301468 container init 2956e555c0254b3e5301b48e9f38148958e6c1259557d363d40aa5cfd6d3fdc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-27T02:32:12.464 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:12 smithi026 podman[112627]: 2024-04-27 02:32:12.325548631 +0000 UTC m=+0.472577996 container start 2956e555c0254b3e5301b48e9f38148958e6c1259557d363d40aa5cfd6d3fdc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-27T02:32:12.464 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:12 smithi026 podman[112627]: 2024-04-27 02:32:12.396959378 +0000 UTC m=+0.543988741 container attach 2956e555c0254b3e5301b48e9f38148958e6c1259557d363d40aa5cfd6d3fdc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-27T02:32:12.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:12 smithi151 ceph-mon[64897]: osdmap e128: 8 total, 7 up, 8 in 2024-04-27T02:32:12.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:12 smithi026 ceph-mon[82260]: osdmap e128: 8 total, 7 up, 8 in 2024-04-27T02:32:12.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:12 smithi026 ceph-mon[87214]: osdmap e128: 8 total, 7 up, 8 in 2024-04-27T02:32:12.853 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:12 smithi026 podman[112627]: 2024-04-27 02:32:12.639536435 +0000 UTC m=+0.786565797 container died 2956e555c0254b3e5301b48e9f38148958e6c1259557d363d40aa5cfd6d3fdc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-27T02:32:13.465 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 podman[112653]: 2024-04-27 02:32:13.218106937 +0000 UTC m=+0.570016884 container remove 2956e555c0254b3e5301b48e9f38148958e6c1259557d363d40aa5cfd6d3fdc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-27T02:32:13.465 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-27T02:32:13.466 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 systemd[1]: Stopped Ceph osd.3 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:32:13.466 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.3.service: Consumed 18.434s CPU time. 2024-04-27T02:32:13.466 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 systemd[1]: Starting Ceph osd.3 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:32:13.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:13 smithi026 ceph-mon[82260]: pgmap v133: 161 pgs: 12 active+undersized+degraded, 13 stale+active+clean, 21 active+undersized, 115 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 41/723 objects degraded (5.671%) 2024-04-27T02:32:13.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:13 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 12 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:13.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:13 smithi026 ceph-mon[87214]: pgmap v133: 161 pgs: 12 active+undersized+degraded, 13 stale+active+clean, 21 active+undersized, 115 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 41/723 objects degraded (5.671%) 2024-04-27T02:32:13.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:13 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 12 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:13.740 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 podman[112758]: 2024-04-27 02:32:13.567514434 +0000 UTC m=+0.018450011 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:32:13.740 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 podman[112758]: 2024-04-27 02:32:13.66906208 +0000 UTC m=+0.119997646 container create 289f6307a7622d4b759090b7920ab15bd045ba66b67503e88bfc760d431b1560 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-27T02:32:13.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:13 smithi151 ceph-mon[64897]: pgmap v133: 161 pgs: 12 active+undersized+degraded, 13 stale+active+clean, 21 active+undersized, 115 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 41/723 objects degraded (5.671%) 2024-04-27T02:32:13.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:13 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 12 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:14.103 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 podman[112758]: 2024-04-27 02:32:13.863348845 +0000 UTC m=+0.314284413 container init 289f6307a7622d4b759090b7920ab15bd045ba66b67503e88bfc760d431b1560 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:32:14.103 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 podman[112758]: 2024-04-27 02:32:13.866538431 +0000 UTC m=+0.317473999 container start 289f6307a7622d4b759090b7920ab15bd045ba66b67503e88bfc760d431b1560 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-b22e2eb) 2024-04-27T02:32:14.103 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:13 smithi026 podman[112758]: 2024-04-27 02:32:13.938575053 +0000 UTC m=+0.389510627 container attach 289f6307a7622d4b759090b7920ab15bd045ba66b67503e88bfc760d431b1560 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate[112772]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate[112772]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 bash[112758]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 bash[112758]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 bash[112758]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 bash[112758]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 bash[112758]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-27T02:32:14.603 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 bash[112758]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-27T02:32:14.604 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 bash[112758]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-27T02:32:14.604 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate[112772]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-27T02:32:14.604 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate[112772]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-27T02:32:14.604 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate[112772]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-27T02:32:14.604 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate[112772]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-27T02:32:14.604 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate[112772]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-27T02:32:14.604 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:14 smithi026 podman[112758]: 2024-04-27 02:32:14.344648613 +0000 UTC m=+0.795584171 container died 289f6307a7622d4b759090b7920ab15bd045ba66b67503e88bfc760d431b1560 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-27T02:32:15.285 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 podman[112894]: 2024-04-27 02:32:15.060012782 +0000 UTC m=+0.710302117 container remove 289f6307a7622d4b759090b7920ab15bd045ba66b67503e88bfc760d431b1560 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-27T02:32:15.285 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 podman[112961]: 2024-04-27 02:32:15.187474173 +0000 UTC m=+0.019129173 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:32:15.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:15 smithi026 ceph-mon[82260]: pgmap v134: 161 pgs: 19 active+undersized+degraded, 7 stale+active+clean, 28 active+undersized, 107 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 69/723 objects degraded (9.544%) 2024-04-27T02:32:15.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:15 smithi026 ceph-mon[87214]: pgmap v134: 161 pgs: 19 active+undersized+degraded, 7 stale+active+clean, 28 active+undersized, 107 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 69/723 objects degraded (9.544%) 2024-04-27T02:32:15.547 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 podman[112961]: 2024-04-27 02:32:15.311175758 +0000 UTC m=+0.142830769 container create a9b308864afa7d3b119146f3177c1053b6c031e0ae4a1735e664a6f7bea603f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-27T02:32:15.547 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 podman[112961]: 2024-04-27 02:32:15.49737754 +0000 UTC m=+0.329032514 container init a9b308864afa7d3b119146f3177c1053b6c031e0ae4a1735e664a6f7bea603f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:32:15.547 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 podman[112961]: 2024-04-27 02:32:15.500773313 +0000 UTC m=+0.332428286 container start a9b308864afa7d3b119146f3177c1053b6c031e0ae4a1735e664a6f7bea603f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-27T02:32:15.547 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 ceph-osd[112980]: -- 172.21.15.26:0/1287409277 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e548991680 con 0x55e54895ac00 2024-04-27T02:32:15.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:15 smithi151 ceph-mon[64897]: pgmap v134: 161 pgs: 19 active+undersized+degraded, 7 stale+active+clean, 28 active+undersized, 107 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 69/723 objects degraded (9.544%) 2024-04-27T02:32:15.852 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 bash[112961]: a9b308864afa7d3b119146f3177c1053b6c031e0ae4a1735e664a6f7bea603f7 2024-04-27T02:32:15.852 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:15 smithi026 systemd[1]: Started Ceph osd.3 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:32:16.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:32:16.048+0000 7fd2492587c0 -1 Falling back to public interface 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[82260]: pgmap v135: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:16.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:16 smithi026 ceph-mon[87214]: pgmap v135: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:17.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:17.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:17.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:17.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:16 smithi151 ceph-mon[64897]: pgmap v135: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:17.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:17 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:17.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:17 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:18.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:17 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:18 smithi026 ceph-mon[82260]: pgmap v136: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:18.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:18 smithi026 ceph-mon[87214]: pgmap v136: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:18.881 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:32:18.768+0000 7fd2492587c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-27T02:32:18.881 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:18 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:32:18.807+0000 7fd2492587c0 -1 osd.3 126 log_to_monitors true 2024-04-27T02:32:19.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:18 smithi151 ceph-mon[64897]: pgmap v136: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:19.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:19 smithi026 ceph-mon[82260]: from='osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:32:19.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:19 smithi026 ceph-mon[82260]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:32:19.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:19 smithi026 ceph-mon[87214]: from='osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:32:19.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:19 smithi026 ceph-mon[87214]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:32:20.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:19 smithi151 ceph-mon[64897]: from='osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:32:20.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:19 smithi151 ceph-mon[64897]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-27T02:32:20.102 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:32:19 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:32:19.660+0000 7fd23a6b0700 -1 osd.3 126 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:32:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:20 smithi151 ceph-mon[64897]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-27T02:32:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:20 smithi151 ceph-mon[64897]: osdmap e129: 8 total, 7 up, 8 in 2024-04-27T02:32:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:20 smithi151 ceph-mon[64897]: from='osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:32:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:20 smithi151 ceph-mon[64897]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:32:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:20 smithi151 ceph-mon[64897]: pgmap v138: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:20 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:20 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[87214]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-27T02:32:21.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[87214]: osdmap e129: 8 total, 7 up, 8 in 2024-04-27T02:32:21.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[87214]: from='osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:32:21.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[87214]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[87214]: pgmap v138: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[82260]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[82260]: osdmap e129: 8 total, 7 up, 8 in 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[82260]: from='osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[82260]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[82260]: pgmap v138: 161 pgs: 25 active+undersized+degraded, 41 active+undersized, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:20 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[82260]: osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999] boot 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[82260]: osdmap e130: 8 total, 8 up, 8 in 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[87214]: osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999] boot 2024-04-27T02:32:21.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[87214]: osdmap e130: 8 total, 8 up, 8 in 2024-04-27T02:32:21.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:32:21.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:21.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:21 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:22.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:21 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:32:22.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:21 smithi151 ceph-mon[64897]: osd.3 [v2:172.21.15.26:6826/3745236999,v1:172.21.15.26:6827/3745236999] boot 2024-04-27T02:32:22.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:21 smithi151 ceph-mon[64897]: osdmap e130: 8 total, 8 up, 8 in 2024-04-27T02:32:22.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:21 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-27T02:32:22.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:21 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:22.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:21 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:22.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:32:22] "GET /metrics HTTP/1.1" 200 38185 "" "Prometheus/2.43.0" 2024-04-27T02:32:23.054 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:22 smithi026 ceph-mon[82260]: osdmap e131: 8 total, 8 up, 8 in 2024-04-27T02:32:23.054 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:22 smithi026 ceph-mon[82260]: pgmap v141: 161 pgs: 18 active+undersized+degraded, 29 active+undersized, 114 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-04-27T02:32:23.054 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:22 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 25 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:23.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:22 smithi026 ceph-mon[87214]: osdmap e131: 8 total, 8 up, 8 in 2024-04-27T02:32:23.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:22 smithi026 ceph-mon[87214]: pgmap v141: 161 pgs: 18 active+undersized+degraded, 29 active+undersized, 114 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-04-27T02:32:23.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:22 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 25 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:23.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:22 smithi151 ceph-mon[64897]: osdmap e131: 8 total, 8 up, 8 in 2024-04-27T02:32:23.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:22 smithi151 ceph-mon[64897]: pgmap v141: 161 pgs: 18 active+undersized+degraded, 29 active+undersized, 114 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-04-27T02:32:23.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:22 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 25 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:25.103 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:32:24.855+0000 7fc7ce81b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-27T02:32:25.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: pgmap v142: 161 pgs: 14 active+undersized+degraded, 25 active+undersized, 122 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-04-27T02:32:25.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:32:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:25.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: pgmap v142: 161 pgs: 14 active+undersized+degraded, 25 active+undersized, 122 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-04-27T02:32:25.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:25.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: pgmap v142: 161 pgs: 14 active+undersized+degraded, 25 active+undersized, 122 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:25.345 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:26.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:25 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:26.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:25 smithi151 ceph-mon[64897]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-27T02:32:26.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:26.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:26.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:26.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[82260]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-27T02:32:26.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:26.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:26.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:26.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[87214]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-27T02:32:26.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:26.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:27.245 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:26 smithi151 ceph-mon[64897]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:32:27.245 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:26 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded) 2024-04-27T02:32:27.245 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:26 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:32:27.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:26 smithi026 ceph-mon[82260]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:32:27.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:26 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded) 2024-04-27T02:32:27.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:26 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:32:27.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:26 smithi026 ceph-mon[87214]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:32:27.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:26 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded) 2024-04-27T02:32:27.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:26 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:32:28.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:27 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:28.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:27 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:28.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:27 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:29.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:28 smithi151 ceph-mon[64897]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:32:29.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:28 smithi026 ceph-mon[82260]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:32:29.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:28 smithi026 ceph-mon[87214]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:32:31.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:30 smithi151 ceph-mon[64897]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-27T02:32:31.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:30 smithi026 ceph-mon[82260]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-27T02:32:31.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:30 smithi026 ceph-mon[87214]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-27T02:32:32.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:32 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:32.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:32 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:32.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:32 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:32.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:32:32] "GET /metrics HTTP/1.1" 200 38189 "" "Prometheus/2.43.0" 2024-04-27T02:32:33.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:33 smithi151 ceph-mon[64897]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-04-27T02:32:33.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:33 smithi026 ceph-mon[87214]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-04-27T02:32:33.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:33 smithi026 ceph-mon[82260]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-04-27T02:32:35.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:35 smithi151 ceph-mon[64897]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-27T02:32:35.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:35 smithi026 ceph-mon[87214]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-27T02:32:35.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:35 smithi026 ceph-mon[82260]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-27T02:32:36.372 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:32:37.051 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:32:37.051 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) 17s ago 28m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:32:37.051 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (6m) 4m ago 28m 80.8M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:32:37.051 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (7m) 17s ago 27m 62.1M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:32:37.051 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (7m) 4m ago 33m 447M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:32:37.051 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (17m) 17s ago 34m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (4m) 17s ago 34m 50.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (5m) 4m ago 33m 36.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 17s ago 33m 41.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) 17s ago 29m 16.3M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (7m) 4m ago 29m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) 17s ago 32m 50.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (2m) 17s ago 32m 47.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (76s) 17s ago 31m 41.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (21s) 17s ago 31m 12.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (31m) 4m ago 31m 57.1M 3248M 17.2.0 e1d6a67b021e 99855dd8b8f0 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (30m) 4m ago 30m 55.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (30m) 4m ago 30m 52.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (30m) 4m ago 30m 56.0M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:32:37.052 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (6m) 4m ago 29m 60.1M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:32:37.053 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (27m) 17s ago 27m 78.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:32:37.053 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (27m) 4m ago 27m 79.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:32:37.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:37 smithi026 ceph-mon[82260]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-27T02:32:37.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:37 smithi026 ceph-mon[82260]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:37.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:37 smithi026 ceph-mon[87214]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-27T02:32:37.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:37 smithi026 ceph-mon[87214]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:37.462 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:32:37.463 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-27T02:32:37.464 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 9 2024-04-27T02:32:37.464 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:32:37.464 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:32:37.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:37 smithi151 ceph-mon[64897]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-27T02:32:37.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:37 smithi151 ceph-mon[64897]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "4/8 daemons upgraded", 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:32:37.820 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:32:38.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:38 smithi151 ceph-mon[64897]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:38.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:38 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:38.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:38 smithi151 ceph-mon[64897]: from='client.54391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:38.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:38 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/3291017519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:38.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[82260]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:38.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:38.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[82260]: from='client.54391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:38.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/3291017519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[87214]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[87214]: from='client.54391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:38 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/3291017519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:32:39.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:39 smithi151 ceph-mon[64897]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:39.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:39 smithi151 ceph-mon[64897]: from='client.54403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:39 smithi026 ceph-mon[82260]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:39 smithi026 ceph-mon[82260]: from='client.54403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:39.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:39 smithi026 ceph-mon[87214]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:39.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:39 smithi026 ceph-mon[87214]: from='client.54403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:32:40.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:40.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:40.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:41.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:41 smithi151 ceph-mon[64897]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:41.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:41 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:41.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:41 smithi151 ceph-mon[64897]: Upgrade: osd.4 is safe to restart 2024-04-27T02:32:41.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:41.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:41.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[82260]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:41.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[82260]: Upgrade: osd.4 is safe to restart 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[87214]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[87214]: Upgrade: osd.4 is safe to restart 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:41.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:42.515 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:32:42] "GET /metrics HTTP/1.1" 200 38194 "" "Prometheus/2.43.0" 2024-04-27T02:32:42.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:42 smithi151 ceph-mon[64897]: Upgrade: Updating osd.4 2024-04-27T02:32:42.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:42.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:32:42.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:42 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:42.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:42 smithi151 ceph-mon[64897]: Deploying daemon osd.4 on smithi151 2024-04-27T02:32:42.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[82260]: Upgrade: Updating osd.4 2024-04-27T02:32:42.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:42.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:32:42.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:42.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[82260]: Deploying daemon osd.4 on smithi151 2024-04-27T02:32:42.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[87214]: Upgrade: Updating osd.4 2024-04-27T02:32:42.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:42.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-27T02:32:42.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:32:42.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:42 smithi026 ceph-mon[87214]: Deploying daemon osd.4 on smithi151 2024-04-27T02:32:43.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:43 smithi151 ceph-mon[64897]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:43.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:43 smithi026 ceph-mon[82260]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:43.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:43 smithi026 ceph-mon[87214]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:44.803 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:44 smithi151 systemd[1]: Stopping Ceph osd.4 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:32:45.093 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[26043]: 2024-04-27T02:32:44.840+0000 7f24d9cfb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:32:45.094 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[26043]: 2024-04-27T02:32:44.840+0000 7f24d9cfb700 -1 osd.4 131 *** Got signal Terminated *** 2024-04-27T02:32:45.094 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[26043]: 2024-04-27T02:32:44.840+0000 7f24d9cfb700 -1 osd.4 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:32:45.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:45 smithi151 ceph-mon[64897]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:45.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:45 smithi151 ceph-mon[64897]: osd.4 marked itself down and dead 2024-04-27T02:32:45.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:45 smithi026 ceph-mon[82260]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:45.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:45 smithi026 ceph-mon[82260]: osd.4 marked itself down and dead 2024-04-27T02:32:45.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:45 smithi026 ceph-mon[87214]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:32:45.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:45 smithi026 ceph-mon[87214]: osd.4 marked itself down and dead 2024-04-27T02:32:46.157 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:45 smithi151 podman[75167]: 2024-04-27 02:32:45.888889216 +0000 UTC m=+1.144718315 container died 99855dd8b8f0f2c8c1bf941564b3ebfb6fc9ae80b94e04ca0e134c9ae53c2c37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, RELEASE=HEAD, name=centos-stream, architecture=x86_64, ceph=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream) 2024-04-27T02:32:46.498 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:46 smithi151 podman[75167]: 2024-04-27 02:32:46.282059966 +0000 UTC m=+1.537889056 container cleanup 99855dd8b8f0f2c8c1bf941564b3ebfb6fc9ae80b94e04ca0e134c9ae53c2c37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, vendor=Red Hat, Inc., version=8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.expose-services=, io.buildah.version=1.19.8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-27T02:32:46.498 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:46 smithi151 bash[75167]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4 2024-04-27T02:32:46.831 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:46 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:32:46.831 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:46 smithi151 ceph-mon[64897]: osdmap e132: 8 total, 7 up, 8 in 2024-04-27T02:32:46.831 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:46 smithi151 ceph-mon[64897]: osdmap e133: 8 total, 7 up, 8 in 2024-04-27T02:32:46.831 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:46 smithi151 podman[75196]: 2024-04-27 02:32:46.524977125 +0000 UTC m=+0.636705670 container remove 99855dd8b8f0f2c8c1bf941564b3ebfb6fc9ae80b94e04ca0e134c9ae53c2c37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, release=754, RELEASE=HEAD, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-27T02:32:46.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:46 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:32:46.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:46 smithi026 ceph-mon[82260]: osdmap e132: 8 total, 7 up, 8 in 2024-04-27T02:32:46.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:46 smithi026 ceph-mon[82260]: osdmap e133: 8 total, 7 up, 8 in 2024-04-27T02:32:46.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:46 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:32:46.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:46 smithi026 ceph-mon[87214]: osdmap e132: 8 total, 7 up, 8 in 2024-04-27T02:32:46.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:46 smithi026 ceph-mon[87214]: osdmap e133: 8 total, 7 up, 8 in 2024-04-27T02:32:47.093 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:46 smithi151 podman[75285]: 2024-04-27 02:32:46.760880731 +0000 UTC m=+0.018556614 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:32:47.094 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:46 smithi151 podman[75285]: 2024-04-27 02:32:46.903293684 +0000 UTC m=+0.160969559 container create 14b31ee3c9f652a75d7403135aa45b2e76cc0a3ed401c608795639ce7df37a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-27T02:32:47.449 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:47 smithi151 podman[75285]: 2024-04-27 02:32:47.170588811 +0000 UTC m=+0.428264680 container init 14b31ee3c9f652a75d7403135aa45b2e76cc0a3ed401c608795639ce7df37a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-27T02:32:47.449 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:47 smithi151 podman[75285]: 2024-04-27 02:32:47.173616366 +0000 UTC m=+0.431292239 container start 14b31ee3c9f652a75d7403135aa45b2e76cc0a3ed401c608795639ce7df37a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-04-27T02:32:47.449 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:47 smithi151 podman[75285]: 2024-04-27 02:32:47.25385816 +0000 UTC m=+0.511534035 container attach 14b31ee3c9f652a75d7403135aa45b2e76cc0a3ed401c608795639ce7df37a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD) 2024-04-27T02:32:47.738 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:47 smithi151 podman[75285]: 2024-04-27 02:32:47.475827834 +0000 UTC m=+0.733503713 container died 14b31ee3c9f652a75d7403135aa45b2e76cc0a3ed401c608795639ce7df37a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-04-27T02:32:47.739 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:47 smithi151 ceph-mon[64897]: pgmap v154: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:32:47.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:47 smithi026 ceph-mon[82260]: pgmap v154: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:32:47.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:47 smithi026 ceph-mon[87214]: pgmap v154: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:32:48.344 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 podman[75312]: 2024-04-27 02:32:48.049125932 +0000 UTC m=+0.567582060 container remove 14b31ee3c9f652a75d7403135aa45b2e76cc0a3ed401c608795639ce7df37a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-27T02:32:48.344 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-27T02:32:48.344 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 systemd[1]: Stopped Ceph osd.4 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:32:48.344 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.4.service: Consumed 16.695s CPU time. 2024-04-27T02:32:48.344 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 systemd[1]: Starting Ceph osd.4 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:32:48.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:48.702 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 podman[75418]: 2024-04-27 02:32:48.377315437 +0000 UTC m=+0.018286215 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:32:48.702 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 podman[75418]: 2024-04-27 02:32:48.50031885 +0000 UTC m=+0.141289627 container create 1715a31690fcf4451ae1ec296039554988e1e8cd07fb99a240d0570dd4e298c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-27T02:32:48.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:48.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:49.094 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 podman[75418]: 2024-04-27 02:32:48.727803013 +0000 UTC m=+0.368773801 container init 1715a31690fcf4451ae1ec296039554988e1e8cd07fb99a240d0570dd4e298c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-27T02:32:49.094 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 podman[75418]: 2024-04-27 02:32:48.731498832 +0000 UTC m=+0.372469619 container start 1715a31690fcf4451ae1ec296039554988e1e8cd07fb99a240d0570dd4e298c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-27T02:32:49.094 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:48 smithi151 podman[75418]: 2024-04-27 02:32:48.81919106 +0000 UTC m=+0.460161839 container attach 1715a31690fcf4451ae1ec296039554988e1e8cd07fb99a240d0570dd4e298c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-27T02:32:49.463 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate[75433]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-27T02:32:49.463 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate[75433]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-27T02:32:49.463 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 bash[75418]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-27T02:32:49.463 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 bash[75418]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 bash[75418]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 bash[75418]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 bash[75418]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 bash[75418]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 bash[75418]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate[75433]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate[75433]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate[75433]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate[75433]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate[75433]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-27T02:32:49.464 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 podman[75418]: 2024-04-27 02:32:49.1993878 +0000 UTC m=+0.840358572 container died 1715a31690fcf4451ae1ec296039554988e1e8cd07fb99a240d0570dd4e298c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True) 2024-04-27T02:32:49.730 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-mon[64897]: pgmap v156: 161 pgs: 11 active+undersized+degraded, 14 stale+active+clean, 21 active+undersized, 115 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-04-27T02:32:49.730 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:49 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 62/723 objects degraded (8.575%), 11 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:49.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:49 smithi026 ceph-mon[82260]: pgmap v156: 161 pgs: 11 active+undersized+degraded, 14 stale+active+clean, 21 active+undersized, 115 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-04-27T02:32:49.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:49 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 62/723 objects degraded (8.575%), 11 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:49.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:49 smithi026 ceph-mon[87214]: pgmap v156: 161 pgs: 11 active+undersized+degraded, 14 stale+active+clean, 21 active+undersized, 115 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-04-27T02:32:49.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:49 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 62/723 objects degraded (8.575%), 11 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:50.068 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 podman[75565]: 2024-04-27 02:32:49.756118217 +0000 UTC m=+0.551611863 container remove 1715a31690fcf4451ae1ec296039554988e1e8cd07fb99a240d0570dd4e298c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-27T02:32:50.068 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 podman[75632]: 2024-04-27 02:32:49.880910754 +0000 UTC m=+0.019070125 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:32:50.068 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:49 smithi151 podman[75632]: 2024-04-27 02:32:49.998896908 +0000 UTC m=+0.137056310 container create 5f13d235f4f4d7f5ddd78cd4b89aadeacd88a0a3b0b35cdd4dcb018170efd03d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-04-27T02:32:50.594 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:50 smithi151 podman[75632]: 2024-04-27 02:32:50.218024439 +0000 UTC m=+0.356183800 container init 5f13d235f4f4d7f5ddd78cd4b89aadeacd88a0a3b0b35cdd4dcb018170efd03d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-27T02:32:50.594 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:50 smithi151 podman[75632]: 2024-04-27 02:32:50.220944287 +0000 UTC m=+0.359103647 container start 5f13d235f4f4d7f5ddd78cd4b89aadeacd88a0a3b0b35cdd4dcb018170efd03d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:32:50.594 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:50 smithi151 ceph-osd[75651]: -- 172.21.15.151:0/1299835312 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c2322cd680 con 0x55c232296c00 2024-04-27T02:32:50.594 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:50 smithi151 bash[75632]: 5f13d235f4f4d7f5ddd78cd4b89aadeacd88a0a3b0b35cdd4dcb018170efd03d 2024-04-27T02:32:50.594 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:50 smithi151 systemd[1]: Started Ceph osd.4 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:32:51.093 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:32:50.769+0000 7f5e63b157c0 -1 Falling back to public interface 2024-04-27T02:32:51.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:51 smithi151 ceph-mon[64897]: pgmap v157: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:51.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:51.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:51.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[82260]: pgmap v157: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:51.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:51.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[87214]: pgmap v157: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:32:52.505 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:32:52] "GET /metrics HTTP/1.1" 200 38194 "" "Prometheus/2.43.0" 2024-04-27T02:32:53.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:53 smithi151 ceph-mon[64897]: pgmap v158: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:53.594 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:32:53.460+0000 7f5e63b157c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-27T02:32:53.594 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:53 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:32:53.491+0000 7f5e63b157c0 -1 osd.4 131 log_to_monitors true 2024-04-27T02:32:53.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:53 smithi026 ceph-mon[82260]: pgmap v158: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:53.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:53 smithi026 ceph-mon[87214]: pgmap v158: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:54.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:54 smithi151 ceph-mon[64897]: from='osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:32:54.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:54 smithi151 ceph-mon[64897]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:32:54.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:54 smithi026 ceph-mon[82260]: from='osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:32:54.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:54 smithi026 ceph-mon[82260]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:32:54.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:54 smithi026 ceph-mon[87214]: from='osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:32:54.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:54 smithi026 ceph-mon[87214]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-27T02:32:55.093 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:32:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:32:54.863+0000 7f5e50f65700 -1 osd.4 131 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:32:55.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: pgmap v159: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-27T02:32:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: from='osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:32:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: osdmap e134: 8 total, 7 up, 8 in 2024-04-27T02:32:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:32:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:55 smithi151 ceph-mon[64897]: from='osd.4 ' entity='osd.4' 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: pgmap v159: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: from='osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: osdmap e134: 8 total, 7 up, 8 in 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:55.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[82260]: from='osd.4 ' entity='osd.4' 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: pgmap v159: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: from='osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: osdmap e134: 8 total, 7 up, 8 in 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:55.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:55 smithi026 ceph-mon[87214]: from='osd.4 ' entity='osd.4' 2024-04-27T02:32:56.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: pgmap v161: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535] boot 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: osdmap e135: 8 total, 8 up, 8 in 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:56.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: pgmap v161: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:56.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:56.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:56.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:32:56.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535] boot 2024-04-27T02:32:56.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: osdmap e135: 8 total, 8 up, 8 in 2024-04-27T02:32:56.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: pgmap v161: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: osd.4 [v2:172.21.15.151:6800/2985349535,v1:172.21.15.151:6801/2985349535] boot 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: osdmap e135: 8 total, 8 up, 8 in 2024-04-27T02:32:57.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-27T02:32:58.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:57 smithi151 ceph-mon[64897]: osdmap e136: 8 total, 8 up, 8 in 2024-04-27T02:32:58.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:57 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:58.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:57 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 122/723 objects degraded (16.874%), 24 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:58.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:57 smithi026 ceph-mon[82260]: osdmap e136: 8 total, 8 up, 8 in 2024-04-27T02:32:58.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:57 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:58.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:57 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 122/723 objects degraded (16.874%), 24 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:58.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:57 smithi026 ceph-mon[87214]: osdmap e136: 8 total, 8 up, 8 in 2024-04-27T02:32:58.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:57 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:32:58.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:57 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 122/723 objects degraded (16.874%), 24 pgs degraded (PG_DEGRADED) 2024-04-27T02:32:59.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:32:58 smithi151 ceph-mon[64897]: pgmap v164: 161 pgs: 23 peering, 16 active+undersized+degraded, 29 active+undersized, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:32:59.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:32:58 smithi026 ceph-mon[82260]: pgmap v164: 161 pgs: 23 peering, 16 active+undersized+degraded, 29 active+undersized, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:32:59.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:32:58 smithi026 ceph-mon[87214]: pgmap v164: 161 pgs: 23 peering, 16 active+undersized+degraded, 29 active+undersized, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-04-27T02:32:59.852 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:32:59 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:32:59.432+0000 7fc7ce81b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-27T02:33:00.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:00.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:00 smithi151 ceph-mon[64897]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-27T02:33:00.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[82260]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:00.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:00.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:00 smithi026 ceph-mon[87214]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-27T02:33:01.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:01 smithi151 ceph-mon[64897]: pgmap v165: 161 pgs: 23 peering, 138 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-27T02:33:01.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:01 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded) 2024-04-27T02:33:01.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:01 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:33:01.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:01 smithi026 ceph-mon[87214]: pgmap v165: 161 pgs: 23 peering, 138 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-27T02:33:01.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:01 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded) 2024-04-27T02:33:01.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:01 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:33:01.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:01 smithi026 ceph-mon[82260]: pgmap v165: 161 pgs: 23 peering, 138 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-27T02:33:01.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:01 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded) 2024-04-27T02:33:01.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:01 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:33:02.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:33:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:33:02] "GET /metrics HTTP/1.1" 200 38050 "" "Prometheus/2.43.0" 2024-04-27T02:33:03.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:03 smithi151 ceph-mon[64897]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:33:03.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:03 smithi026 ceph-mon[82260]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:33:03.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:03 smithi026 ceph-mon[87214]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-27T02:33:05.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:05 smithi151 ceph-mon[64897]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:33:05.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:05 smithi026 ceph-mon[82260]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:33:05.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:05 smithi026 ceph-mon[87214]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:33:07.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:07 smithi151 ceph-mon[64897]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:33:07.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:07 smithi026 ceph-mon[87214]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:33:07.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:07 smithi026 ceph-mon[82260]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:33:08.195 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:33:08.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:08 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:08.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:08 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:08.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:08 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:08.871 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:33:08.871 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (7m) 48s ago 28m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:33:08.871 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (6m) 14s ago 28m 80.2M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:33:08.871 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (7m) 48s ago 28m 62.1M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:33:08.871 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (8m) 14s ago 33m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (18m) 48s ago 35m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (4m) 48s ago 35m 50.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (5m) 14s ago 33m 45.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) 48s ago 33m 41.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) 48s ago 29m 16.3M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (7m) 14s ago 29m 15.4M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (3m) 48s ago 33m 50.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (2m) 48s ago 32m 47.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (108s) 48s ago 32m 41.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (53s) 48s ago 32m 12.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (19s) 14s ago 31m 11.9M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (31m) 14s ago 31m 59.6M 3248M 17.2.0 e1d6a67b021e a2832d5c7b67 2024-04-27T02:33:08.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (30m) 14s ago 30m 55.7M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:33:08.873 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (30m) 14s ago 30m 59.6M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:33:08.873 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (7m) 14s ago 29m 70.9M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:33:08.873 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (28m) 48s ago 28m 78.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:33:08.873 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (28m) 14s ago 28m 80.8M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:33:09.284 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:33:09.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:33:09.284 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:33:09.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:09.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:33:09.285 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:33:09.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:09.285 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:33:09.285 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-27T02:33:09.285 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 5 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 10 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:33:09.286 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:33:09.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:09 smithi151 ceph-mon[64897]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-27T02:33:09.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:09 smithi151 ceph-mon[64897]: from='client.54412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:09.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:09 smithi151 ceph-mon[64897]: from='client.44481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:09.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:09 smithi026 ceph-mon[87214]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-27T02:33:09.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:09 smithi026 ceph-mon[87214]: from='client.54412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:09.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:09 smithi026 ceph-mon[87214]: from='client.44481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:09 smithi026 ceph-mon[82260]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-27T02:33:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:09 smithi026 ceph-mon[82260]: from='client.54412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:09 smithi026 ceph-mon[82260]: from='client.44481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "5/8 daemons upgraded", 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:33:09.627 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:33:10.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:10 smithi151 ceph-mon[64897]: from='client.44487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:10.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:10 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/1403687758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:10.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:10 smithi026 ceph-mon[82260]: from='client.44487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:10.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:10 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/1403687758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:10.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:10 smithi026 ceph-mon[87214]: from='client.44487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:10.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:10 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/1403687758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:11.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:11 smithi151 ceph-mon[64897]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:11 smithi151 ceph-mon[64897]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:11.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[82260]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[82260]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:11.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[87214]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:11.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[87214]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:11.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:11.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:11.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:12.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:33:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:33:12] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-04-27T02:33:13.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:13 smithi151 ceph-mon[64897]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:13.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:13 smithi026 ceph-mon[82260]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:13.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:13 smithi026 ceph-mon[87214]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:15.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:15 smithi151 ceph-mon[64897]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:15.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:15 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:15.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:15 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:15.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:15 smithi151 ceph-mon[64897]: Upgrade: osd.5 is safe to restart 2024-04-27T02:33:15.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[87214]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:15.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:15.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:15.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[87214]: Upgrade: osd.5 is safe to restart 2024-04-27T02:33:15.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[82260]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:15.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:15.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-27T02:33:15.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:15 smithi026 ceph-mon[82260]: Upgrade: osd.5 is safe to restart 2024-04-27T02:33:17.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:16 smithi151 ceph-mon[64897]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:17.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:16 smithi151 ceph-mon[64897]: Upgrade: Updating osd.5 2024-04-27T02:33:17.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:17.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:33:17.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:16 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:17.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:16 smithi151 ceph-mon[64897]: Deploying daemon osd.5 on smithi151 2024-04-27T02:33:17.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[82260]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:17.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[82260]: Upgrade: Updating osd.5 2024-04-27T02:33:17.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:17.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[82260]: Deploying daemon osd.5 on smithi151 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[87214]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[87214]: Upgrade: Updating osd.5 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:16 smithi026 ceph-mon[87214]: Deploying daemon osd.5 on smithi151 2024-04-27T02:33:18.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:17 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:18.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:17 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:18.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:17 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:19.241 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:18 smithi151 ceph-mon[64897]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:19.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:18 smithi026 ceph-mon[82260]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:19.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:18 smithi026 ceph-mon[87214]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:19.511 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:19 smithi151 systemd[1]: Stopping Ceph osd.5 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:33:19.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[28995]: 2024-04-27T02:33:19.536+0000 7f9226a57700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:33:19.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[28995]: 2024-04-27T02:33:19.536+0000 7f9226a57700 -1 osd.5 136 *** Got signal Terminated *** 2024-04-27T02:33:19.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[28995]: 2024-04-27T02:33:19.536+0000 7f9226a57700 -1 osd.5 136 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:33:20.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:19 smithi151 ceph-mon[64897]: osd.5 marked itself down and dead 2024-04-27T02:33:20.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:19 smithi026 ceph-mon[82260]: osd.5 marked itself down and dead 2024-04-27T02:33:20.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:19 smithi026 ceph-mon[87214]: osd.5 marked itself down and dead 2024-04-27T02:33:20.719 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:20 smithi151 podman[80198]: 2024-04-27 02:33:20.46882509 +0000 UTC m=+1.057187698 container died a2832d5c7b67d32a24042049c505e2ce7ca2aec8ddff7b35424a4136efa78e6b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.expose-services=, GIT_CLEAN=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-27T02:33:21.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:20 smithi151 ceph-mon[64897]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:20 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:33:21.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:20 smithi151 ceph-mon[64897]: osdmap e137: 8 total, 7 up, 8 in 2024-04-27T02:33:21.094 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:20 smithi151 podman[80198]: 2024-04-27 02:33:20.844731546 +0000 UTC m=+1.433094152 container cleanup a2832d5c7b67d32a24042049c505e2ce7ca2aec8ddff7b35424a4136efa78e6b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, name=centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.openshift.expose-services=, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0) 2024-04-27T02:33:21.094 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:20 smithi151 bash[80198]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5 2024-04-27T02:33:21.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:20 smithi026 ceph-mon[82260]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:21.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:20 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:33:21.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:20 smithi026 ceph-mon[82260]: osdmap e137: 8 total, 7 up, 8 in 2024-04-27T02:33:21.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:20 smithi026 ceph-mon[87214]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:21.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:20 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:33:21.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:20 smithi026 ceph-mon[87214]: osdmap e137: 8 total, 7 up, 8 in 2024-04-27T02:33:21.441 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:21 smithi151 podman[80215]: 2024-04-27 02:33:21.120896466 +0000 UTC m=+0.645993853 container remove a2832d5c7b67d32a24042049c505e2ce7ca2aec8ddff7b35424a4136efa78e6b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, io.openshift.expose-services=, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, release=754, vendor=Red Hat, Inc., name=centos-stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, ceph=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0) 2024-04-27T02:33:21.693 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:21 smithi151 podman[80308]: 2024-04-27 02:33:21.370888317 +0000 UTC m=+0.018359413 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:33:21.693 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:21 smithi151 podman[80308]: 2024-04-27 02:33:21.480151682 +0000 UTC m=+0.127622765 container create f95be5b78334706880f7b483abaf3e252c4c4ff4b5c06d88ed8db9d520671d7f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:33:21.693 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:21 smithi151 podman[80308]: 2024-04-27 02:33:21.7165179 +0000 UTC m=+0.363988993 container init f95be5b78334706880f7b483abaf3e252c4c4ff4b5c06d88ed8db9d520671d7f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:33:21.694 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:21 smithi151 podman[80308]: 2024-04-27 02:33:21.720088008 +0000 UTC m=+0.367559100 container start f95be5b78334706880f7b483abaf3e252c4c4ff4b5c06d88ed8db9d520671d7f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:33:21.967 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:21 smithi151 podman[80308]: 2024-04-27 02:33:21.824508542 +0000 UTC m=+0.471979646 container attach f95be5b78334706880f7b483abaf3e252c4c4ff4b5c06d88ed8db9d520671d7f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-27T02:33:22.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:21 smithi026 ceph-mon[82260]: osdmap e138: 8 total, 7 up, 8 in 2024-04-27T02:33:22.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:21 smithi026 ceph-mon[87214]: osdmap e138: 8 total, 7 up, 8 in 2024-04-27T02:33:22.254 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:33:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:33:22] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-04-27T02:33:22.295 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:22 smithi151 podman[80308]: 2024-04-27 02:33:22.024870246 +0000 UTC m=+0.672341329 container died f95be5b78334706880f7b483abaf3e252c4c4ff4b5c06d88ed8db9d520671d7f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:33:22.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:21 smithi151 ceph-mon[64897]: osdmap e138: 8 total, 7 up, 8 in 2024-04-27T02:33:22.845 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:22 smithi151 podman[80334]: 2024-04-27 02:33:22.586119235 +0000 UTC m=+0.552645259 container remove f95be5b78334706880f7b483abaf3e252c4c4ff4b5c06d88ed8db9d520671d7f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-27T02:33:22.846 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:22 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-27T02:33:22.846 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:22 smithi151 systemd[1]: Stopped Ceph osd.5 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:33:22.846 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:22 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.5.service: Consumed 15.913s CPU time. 2024-04-27T02:33:22.846 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:22 smithi151 systemd[1]: Starting Ceph osd.5 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:33:23.238 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:22 smithi151 ceph-mon[64897]: pgmap v178: 161 pgs: 7 active+undersized+degraded, 17 stale+active+clean, 8 active+undersized, 129 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 22/723 objects degraded (3.043%) 2024-04-27T02:33:23.238 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:22 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 22/723 objects degraded (3.043%), 7 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:23.238 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 podman[80440]: 2024-04-27 02:33:22.932278139 +0000 UTC m=+0.018503235 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:33:23.238 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 podman[80440]: 2024-04-27 02:33:23.045628276 +0000 UTC m=+0.131853342 container create 62dfb8076a683d9945d16510486795a04e4eae0bc9cfb525f73097e36862dbd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-27T02:33:23.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:22 smithi026 ceph-mon[87214]: pgmap v178: 161 pgs: 7 active+undersized+degraded, 17 stale+active+clean, 8 active+undersized, 129 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 22/723 objects degraded (3.043%) 2024-04-27T02:33:23.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:22 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 22/723 objects degraded (3.043%), 7 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:23.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:22 smithi026 ceph-mon[82260]: pgmap v178: 161 pgs: 7 active+undersized+degraded, 17 stale+active+clean, 8 active+undersized, 129 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 22/723 objects degraded (3.043%) 2024-04-27T02:33:23.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:22 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 22/723 objects degraded (3.043%), 7 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:23.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 podman[80440]: 2024-04-27 02:33:23.264705146 +0000 UTC m=+0.350930212 container init 62dfb8076a683d9945d16510486795a04e4eae0bc9cfb525f73097e36862dbd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:33:23.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 podman[80440]: 2024-04-27 02:33:23.267931972 +0000 UTC m=+0.354157042 container start 62dfb8076a683d9945d16510486795a04e4eae0bc9cfb525f73097e36862dbd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:33:23.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 podman[80440]: 2024-04-27 02:33:23.35648149 +0000 UTC m=+0.442706572 container attach 62dfb8076a683d9945d16510486795a04e4eae0bc9cfb525f73097e36862dbd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb) 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate[80455]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate[80455]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 bash[80440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 bash[80440]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 bash[80440]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 bash[80440]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 bash[80440]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 bash[80440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-27T02:33:24.067 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 bash[80440]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-27T02:33:24.068 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate[80455]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-27T02:33:24.068 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate[80455]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-27T02:33:24.068 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate[80455]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-27T02:33:24.068 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate[80455]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-27T02:33:24.068 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate[80455]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-27T02:33:24.068 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:23 smithi151 podman[80440]: 2024-04-27 02:33:23.738624762 +0000 UTC m=+0.824849847 container died 62dfb8076a683d9945d16510486795a04e4eae0bc9cfb525f73097e36862dbd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-27T02:33:24.699 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 podman[80587]: 2024-04-27 02:33:24.410129612 +0000 UTC m=+0.665915945 container remove 62dfb8076a683d9945d16510486795a04e4eae0bc9cfb525f73097e36862dbd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5) 2024-04-27T02:33:24.699 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 podman[80652]: 2024-04-27 02:33:24.526318563 +0000 UTC m=+0.018248283 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:33:24.699 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 podman[80652]: 2024-04-27 02:33:24.627786637 +0000 UTC m=+0.119716433 container create 4d15baed129ee20422317e43200525054c008649d637420220bcdcc6c90aaf50 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:33:25.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:24 smithi151 ceph-mon[64897]: pgmap v179: 161 pgs: 11 active+undersized+degraded, 16 stale+active+clean, 10 active+undersized, 124 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 34/723 objects degraded (4.703%) 2024-04-27T02:33:25.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:25.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:25.094 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 podman[80652]: 2024-04-27 02:33:24.796599971 +0000 UTC m=+0.288529691 container init 4d15baed129ee20422317e43200525054c008649d637420220bcdcc6c90aaf50 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-27T02:33:25.094 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 podman[80652]: 2024-04-27 02:33:24.799540273 +0000 UTC m=+0.291469993 container start 4d15baed129ee20422317e43200525054c008649d637420220bcdcc6c90aaf50 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-27T02:33:25.094 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 ceph-osd[80671]: -- 172.21.15.151:0/1169394142 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aeb438d680 con 0x55aeb4357000 2024-04-27T02:33:25.094 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 bash[80652]: 4d15baed129ee20422317e43200525054c008649d637420220bcdcc6c90aaf50 2024-04-27T02:33:25.094 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:24 smithi151 systemd[1]: Started Ceph osd.5 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:33:25.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:24 smithi026 ceph-mon[87214]: pgmap v179: 161 pgs: 11 active+undersized+degraded, 16 stale+active+clean, 10 active+undersized, 124 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 34/723 objects degraded (4.703%) 2024-04-27T02:33:25.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:25.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:25.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:24 smithi026 ceph-mon[82260]: pgmap v179: 161 pgs: 11 active+undersized+degraded, 16 stale+active+clean, 10 active+undersized, 124 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 34/723 objects degraded (4.703%) 2024-04-27T02:33:25.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:25.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:25.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:33:25.336+0000 7f74a874c7c0 -1 Falling back to public interface 2024-04-27T02:33:26.041 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:26.041 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:26.041 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:26.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:26.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:26.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:26.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:26.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:26.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:27.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:27 smithi151 ceph-mon[64897]: pgmap v180: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:27.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:26 smithi026 ceph-mon[82260]: pgmap v180: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:27.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:27 smithi026 ceph-mon[87214]: pgmap v180: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:28.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:28 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:28.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:28 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 81/723 objects degraded (11.203%), 21 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:28.343 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:33:28.076+0000 7f74a874c7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-27T02:33:28.343 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:33:28.101+0000 7f74a874c7c0 -1 osd.5 136 log_to_monitors true 2024-04-27T02:33:28.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:28 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:28.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:28 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 81/723 objects degraded (11.203%), 21 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:28.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:28 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:28.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:28 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 81/723 objects degraded (11.203%), 21 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:29.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:29 smithi151 ceph-mon[64897]: pgmap v181: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:29.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:29 smithi151 ceph-mon[64897]: from='osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:33:29.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:29 smithi151 ceph-mon[64897]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:33:29.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:29 smithi026 ceph-mon[82260]: pgmap v181: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:29.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:29 smithi026 ceph-mon[82260]: from='osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:33:29.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:29 smithi026 ceph-mon[82260]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:33:29.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:29 smithi026 ceph-mon[87214]: pgmap v181: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:29.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:29 smithi026 ceph-mon[87214]: from='osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:33:29.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:29 smithi026 ceph-mon[87214]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-27T02:33:29.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:33:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:33:29.512+0000 7f7495b9c700 -1 osd.5 136 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:33:30.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:30 smithi151 ceph-mon[64897]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-27T02:33:30.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:30 smithi151 ceph-mon[64897]: osdmap e139: 8 total, 7 up, 8 in 2024-04-27T02:33:30.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:30 smithi151 ceph-mon[64897]: from='osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:33:30.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:30 smithi151 ceph-mon[64897]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:33:30.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:30.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:30.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:30 smithi151 ceph-mon[64897]: from='osd.5 ' entity='osd.5' 2024-04-27T02:33:30.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[82260]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-27T02:33:30.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[82260]: osdmap e139: 8 total, 7 up, 8 in 2024-04-27T02:33:30.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[82260]: from='osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:33:30.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[82260]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:33:30.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[82260]: from='osd.5 ' entity='osd.5' 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[87214]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[87214]: osdmap e139: 8 total, 7 up, 8 in 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[87214]: from='osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[87214]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:30.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:30 smithi026 ceph-mon[87214]: from='osd.5 ' entity='osd.5' 2024-04-27T02:33:31.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:31 smithi151 ceph-mon[64897]: pgmap v183: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:31.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:31 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:31.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:31 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:31.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:31 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:33:31.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:31 smithi151 ceph-mon[64897]: osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128] boot 2024-04-27T02:33:31.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:31 smithi151 ceph-mon[64897]: osdmap e140: 8 total, 8 up, 8 in 2024-04-27T02:33:31.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:31 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:33:31.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[82260]: pgmap v183: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[82260]: osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128] boot 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[82260]: osdmap e140: 8 total, 8 up, 8 in 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[87214]: pgmap v183: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:33:31.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[87214]: osd.5 [v2:172.21.15.151:6808/3476506128,v1:172.21.15.151:6809/3476506128] boot 2024-04-27T02:33:31.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[87214]: osdmap e140: 8 total, 8 up, 8 in 2024-04-27T02:33:31.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:31 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-27T02:33:32.481 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:33:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:33:32] "GET /metrics HTTP/1.1" 200 38204 "" "Prometheus/2.43.0" 2024-04-27T02:33:32.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:32 smithi151 ceph-mon[64897]: osdmap e141: 8 total, 8 up, 8 in 2024-04-27T02:33:32.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:32 smithi026 ceph-mon[82260]: osdmap e141: 8 total, 8 up, 8 in 2024-04-27T02:33:32.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:32 smithi026 ceph-mon[87214]: osdmap e141: 8 total, 8 up, 8 in 2024-04-27T02:33:33.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:33 smithi151 ceph-mon[64897]: pgmap v186: 161 pgs: 17 active+undersized+degraded, 32 active+undersized, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-04-27T02:33:33.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:33 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 17 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:33.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:33 smithi026 ceph-mon[82260]: pgmap v186: 161 pgs: 17 active+undersized+degraded, 32 active+undersized, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-04-27T02:33:33.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:33 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 17 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:33.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:33 smithi026 ceph-mon[87214]: pgmap v186: 161 pgs: 17 active+undersized+degraded, 32 active+undersized, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-04-27T02:33:33.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:33 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 17 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:34.352 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:33:34.030+0000 7fc7ce81b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-27T02:33:35.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: pgmap v187: 161 pgs: 14 active+undersized+degraded, 30 active+undersized, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-04-27T02:33:35.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[87214]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: pgmap v187: 161 pgs: 14 active+undersized+degraded, 30 active+undersized, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:35.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:34 smithi026 ceph-mon[82260]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-27T02:33:35.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: pgmap v187: 161 pgs: 14 active+undersized+degraded, 30 active+undersized, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:35.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:34 smithi151 ceph-mon[64897]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-27T02:33:36.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:35 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 14 pgs degraded) 2024-04-27T02:33:36.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:35 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:33:36.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:35 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 14 pgs degraded) 2024-04-27T02:33:36.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:35 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:33:36.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:35 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 14 pgs degraded) 2024-04-27T02:33:36.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:35 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:33:37.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:36 smithi151 ceph-mon[64897]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:37.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:36 smithi026 ceph-mon[87214]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:37.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:36 smithi026 ceph-mon[82260]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:38.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:37 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:38.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:37 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:38.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:37 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:39.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:38 smithi151 ceph-mon[64897]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-27T02:33:39.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:38 smithi026 ceph-mon[82260]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-27T02:33:39.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:38 smithi026 ceph-mon[87214]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-27T02:33:40.005 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (7m) 80s ago 29m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (7m) 11s ago 29m 80.1M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (8m) 80s ago 28m 62.1M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (8m) 11s ago 34m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (18m) 80s ago 35m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) 80s ago 35m 50.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (6m) 11s ago 34m 47.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) 80s ago 34m 41.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (8m) 80s ago 30m 16.3M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (8m) 11s ago 30m 15.3M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:33:40.687 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (3m) 80s ago 33m 50.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (3m) 80s ago 33m 47.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (2m) 80s ago 33m 41.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (85s) 80s ago 32m 12.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (50s) 11s ago 32m 51.7M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (16s) 11s ago 31m 12.6M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (31m) 11s ago 31m 56.6M 3248M 17.2.0 e1d6a67b021e c11f6399e640 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (31m) 11s ago 31m 60.1M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (7m) 11s ago 30m 70.9M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (28m) 80s ago 28m 78.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:33:40.688 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (28m) 11s ago 28m 80.9M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:33:41.100 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:33:41.101 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:33:41.101 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:33:41.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:41.101 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:33:41.101 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:33:41.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 6 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 11 2024-04-27T02:33:41.102 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:33:41.103 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:33:41.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[82260]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-04-27T02:33:41.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[82260]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:41.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[82260]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:41.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:41.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:41.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[87214]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-04-27T02:33:41.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[87214]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:41.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[87214]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:41.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:41.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:41.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:40 smithi151 ceph-mon[64897]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-04-27T02:33:41.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:40 smithi151 ceph-mon[64897]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:41.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:40 smithi151 ceph-mon[64897]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:41.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:41.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:41.447 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:33:41.447 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:33:41.447 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:33:41.448 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-27T02:33:41.448 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:33:41.448 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "6/8 daemons upgraded", 2024-04-27T02:33:41.448 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:33:41.448 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:33:41.448 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[82260]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2550112383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[82260]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[87214]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2550112383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:42.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:41 smithi026 ceph-mon[87214]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:42.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:41 smithi151 ceph-mon[64897]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:42.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:41 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2550112383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:33:42.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:42.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:41 smithi151 ceph-mon[64897]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:33:42.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:33:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:33:42] "GET /metrics HTTP/1.1" 200 38208 "" "Prometheus/2.43.0" 2024-04-27T02:33:43.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:43 smithi151 ceph-mon[64897]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-04-27T02:33:43.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:43 smithi026 ceph-mon[82260]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-04-27T02:33:43.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:43 smithi026 ceph-mon[87214]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-04-27T02:33:45.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:45 smithi151 ceph-mon[64897]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:33:45.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:45 smithi026 ceph-mon[82260]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:33:45.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:45 smithi026 ceph-mon[87214]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-27T02:33:47.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:47 smithi026 ceph-mon[82260]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:33:47.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:47 smithi026 ceph-mon[87214]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:33:47.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:47 smithi151 ceph-mon[64897]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:33:48.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:48.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:48.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:49.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:49 smithi151 ceph-mon[64897]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:49.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:49 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:49.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:49 smithi026 ceph-mon[87214]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:49.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:49 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:49.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:49 smithi026 ceph-mon[82260]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:49.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:49 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:50.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:50 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:50.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:50 smithi151 ceph-mon[64897]: Upgrade: osd.6 is safe to restart 2024-04-27T02:33:50.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:50 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:50.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:50 smithi026 ceph-mon[82260]: Upgrade: osd.6 is safe to restart 2024-04-27T02:33:50.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:50 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-27T02:33:50.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:50 smithi026 ceph-mon[87214]: Upgrade: osd.6 is safe to restart 2024-04-27T02:33:51.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:51 smithi151 ceph-mon[64897]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:51.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:51.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:33:51.489 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:51 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[82260]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:33:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[87214]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:33:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-27T02:33:51.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:51 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:33:52.503 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:52 smithi026 ceph-mon[82260]: Upgrade: Updating osd.6 2024-04-27T02:33:52.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:52 smithi026 ceph-mon[82260]: Deploying daemon osd.6 on smithi151 2024-04-27T02:33:52.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:52 smithi026 ceph-mon[87214]: Upgrade: Updating osd.6 2024-04-27T02:33:52.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:52 smithi026 ceph-mon[87214]: Deploying daemon osd.6 on smithi151 2024-04-27T02:33:52.504 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:33:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:33:52] "GET /metrics HTTP/1.1" 200 38208 "" "Prometheus/2.43.0" 2024-04-27T02:33:52.531 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:52 smithi151 ceph-mon[64897]: Upgrade: Updating osd.6 2024-04-27T02:33:52.531 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:52 smithi151 ceph-mon[64897]: Deploying daemon osd.6 on smithi151 2024-04-27T02:33:53.482 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:53 smithi151 ceph-mon[64897]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:53.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:53 smithi026 ceph-mon[82260]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:53.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:53 smithi026 ceph-mon[87214]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:54.070 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:53 smithi151 systemd[1]: Stopping Ceph osd.6 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:33:54.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:54 smithi151 ceph-mon[64897]: osd.6 marked itself down and dead 2024-04-27T02:33:54.343 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[31876]: 2024-04-27T02:33:54.121+0000 7f92d5707700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:33:54.344 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[31876]: 2024-04-27T02:33:54.121+0000 7f92d5707700 -1 osd.6 141 *** Got signal Terminated *** 2024-04-27T02:33:54.344 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:54 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[31876]: 2024-04-27T02:33:54.121+0000 7f92d5707700 -1 osd.6 141 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:33:54.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:54 smithi026 ceph-mon[82260]: osd.6 marked itself down and dead 2024-04-27T02:33:54.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:54 smithi026 ceph-mon[87214]: osd.6 marked itself down and dead 2024-04-27T02:33:54.888 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:54 smithi151 podman[85044]: 2024-04-27 02:33:54.630218792 +0000 UTC m=+0.615801917 container died c11f6399e640c38bce4974a612505e77ae77e91f52ad1beb5027baaec42f4b28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, version=8, ceph=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-27T02:33:55.216 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:54 smithi151 podman[85044]: 2024-04-27 02:33:54.996669056 +0000 UTC m=+0.982252182 container cleanup c11f6399e640c38bce4974a612505e77ae77e91f52ad1beb5027baaec42f4b28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-type=git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, release=754, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8) 2024-04-27T02:33:55.217 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:54 smithi151 bash[85044]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6 2024-04-27T02:33:55.593 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:55 smithi151 podman[85072]: 2024-04-27 02:33:55.348117958 +0000 UTC m=+0.713509987 container remove c11f6399e640c38bce4974a612505e77ae77e91f52ad1beb5027baaec42f4b28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, io.k8s.display-name=CentOS Stream 8, release=754, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.openshift.expose-services=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-type=git) 2024-04-27T02:33:55.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:55 smithi151 ceph-mon[64897]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:55.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:55 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:33:55.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:55 smithi151 ceph-mon[64897]: osdmap e142: 8 total, 7 up, 8 in 2024-04-27T02:33:55.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:55 smithi026 ceph-mon[82260]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:55.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:55 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:33:55.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:55 smithi026 ceph-mon[82260]: osdmap e142: 8 total, 7 up, 8 in 2024-04-27T02:33:55.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:55 smithi026 ceph-mon[87214]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:33:55.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:55 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:33:55.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:55 smithi026 ceph-mon[87214]: osdmap e142: 8 total, 7 up, 8 in 2024-04-27T02:33:56.008 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:55 smithi151 podman[85164]: 2024-04-27 02:33:55.668006448 +0000 UTC m=+0.018781202 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:33:56.008 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:55 smithi151 podman[85164]: 2024-04-27 02:33:55.774103113 +0000 UTC m=+0.124877852 container create d2d7035b82a6c73bbdc81441d7d2fb17b07e8e37aca9948f18ae9c4cf47a6c8d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, ceph=True, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:33:56.008 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:55 smithi151 podman[85164]: 2024-04-27 02:33:55.951524394 +0000 UTC m=+0.302299134 container init d2d7035b82a6c73bbdc81441d7d2fb17b07e8e37aca9948f18ae9c4cf47a6c8d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-27T02:33:56.008 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:55 smithi151 podman[85164]: 2024-04-27 02:33:55.954626126 +0000 UTC m=+0.305400868 container start d2d7035b82a6c73bbdc81441d7d2fb17b07e8e37aca9948f18ae9c4cf47a6c8d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-27T02:33:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:56 smithi151 ceph-mon[64897]: osdmap e143: 8 total, 7 up, 8 in 2024-04-27T02:33:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:56.344 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:56 smithi151 podman[85164]: 2024-04-27 02:33:56.034626763 +0000 UTC m=+0.385401512 container attach d2d7035b82a6c73bbdc81441d7d2fb17b07e8e37aca9948f18ae9c4cf47a6c8d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:33:56.344 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:56 smithi151 podman[85164]: 2024-04-27 02:33:56.246817082 +0000 UTC m=+0.597591831 container died d2d7035b82a6c73bbdc81441d7d2fb17b07e8e37aca9948f18ae9c4cf47a6c8d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-27T02:33:56.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:56 smithi026 ceph-mon[82260]: osdmap e143: 8 total, 7 up, 8 in 2024-04-27T02:33:56.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:56.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:56.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:56 smithi026 ceph-mon[87214]: osdmap e143: 8 total, 7 up, 8 in 2024-04-27T02:33:56.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:56.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:33:57.094 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:56 smithi151 podman[85191]: 2024-04-27 02:33:56.830198598 +0000 UTC m=+0.576828483 container remove d2d7035b82a6c73bbdc81441d7d2fb17b07e8e37aca9948f18ae9c4cf47a6c8d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-27T02:33:57.094 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:56 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-27T02:33:57.094 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:56 smithi151 systemd[1]: Stopped Ceph osd.6 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:33:57.094 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:56 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.6.service: Consumed 14.598s CPU time. 2024-04-27T02:33:57.094 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 systemd[1]: Starting Ceph osd.6 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:33:57.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:57 smithi026 ceph-mon[82260]: pgmap v200: 161 pgs: 26 peering, 6 stale+active+clean, 129 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-27T02:33:57.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:57 smithi026 ceph-mon[82260]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-27T02:33:57.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:57 smithi026 ceph-mon[87214]: pgmap v200: 161 pgs: 26 peering, 6 stale+active+clean, 129 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-27T02:33:57.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:57 smithi026 ceph-mon[87214]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-27T02:33:57.385 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-mon[64897]: pgmap v200: 161 pgs: 26 peering, 6 stale+active+clean, 129 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-27T02:33:57.385 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-mon[64897]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-27T02:33:57.386 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 podman[85294]: 2024-04-27 02:33:57.191723939 +0000 UTC m=+0.018316323 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:33:57.386 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 podman[85294]: 2024-04-27 02:33:57.29776805 +0000 UTC m=+0.124360420 container create 56f01d9ab115644f2b03d48cd1b2b8f4a982858d024c4013ec1fbcffa4895df0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-27T02:33:57.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 podman[85294]: 2024-04-27 02:33:57.500306517 +0000 UTC m=+0.326898892 container init 56f01d9ab115644f2b03d48cd1b2b8f4a982858d024c4013ec1fbcffa4895df0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:33:57.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 podman[85294]: 2024-04-27 02:33:57.503578353 +0000 UTC m=+0.330170728 container start 56f01d9ab115644f2b03d48cd1b2b8f4a982858d024c4013ec1fbcffa4895df0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-27T02:33:57.844 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 podman[85294]: 2024-04-27 02:33:57.583292523 +0000 UTC m=+0.409884902 container attach 56f01d9ab115644f2b03d48cd1b2b8f4a982858d024c4013ec1fbcffa4895df0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, ceph=True) 2024-04-27T02:33:58.228 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate[85308]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-27T02:33:58.229 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate[85308]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-27T02:33:58.229 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 bash[85294]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-27T02:33:58.229 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 bash[85294]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-27T02:33:58.229 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 bash[85294]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-27T02:33:58.229 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 bash[85294]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 bash[85294]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 bash[85294]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 bash[85294]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate[85308]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate[85308]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate[85308]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate[85308]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate[85308]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-27T02:33:58.230 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:57 smithi151 podman[85294]: 2024-04-27 02:33:57.970869806 +0000 UTC m=+0.797462178 container died 56f01d9ab115644f2b03d48cd1b2b8f4a982858d024c4013ec1fbcffa4895df0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-27T02:33:58.502 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:58 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:58.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:58 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:58.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:58 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:33:58.834 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:58 smithi151 podman[85425]: 2024-04-27 02:33:58.528726944 +0000 UTC m=+0.551491203 container remove 56f01d9ab115644f2b03d48cd1b2b8f4a982858d024c4013ec1fbcffa4895df0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0) 2024-04-27T02:33:58.834 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:58 smithi151 podman[85491]: 2024-04-27 02:33:58.648908001 +0000 UTC m=+0.018006389 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:33:58.834 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:58 smithi151 podman[85491]: 2024-04-27 02:33:58.7545342 +0000 UTC m=+0.123632590 container create 88ba76966dafe2f721e99abc4dbb5fe237b30e0ccd87b42519b412180f688704 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-27T02:33:59.093 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:58 smithi151 podman[85491]: 2024-04-27 02:33:58.956758041 +0000 UTC m=+0.325856430 container init 88ba76966dafe2f721e99abc4dbb5fe237b30e0ccd87b42519b412180f688704 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:33:59.093 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:58 smithi151 podman[85491]: 2024-04-27 02:33:58.95982961 +0000 UTC m=+0.328928002 container start 88ba76966dafe2f721e99abc4dbb5fe237b30e0ccd87b42519b412180f688704 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-27T02:33:59.093 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:58 smithi151 ceph-osd[85509]: -- 172.21.15.151:0/3681953502 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555e335eb680 con 0x555e335b5400 2024-04-27T02:33:59.093 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:59 smithi151 bash[85491]: 88ba76966dafe2f721e99abc4dbb5fe237b30e0ccd87b42519b412180f688704 2024-04-27T02:33:59.094 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:59 smithi151 systemd[1]: Started Ceph osd.6 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:33:59.593 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:33:59 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:33:59.499+0000 7f6d3fbe77c0 -1 Falling back to public interface 2024-04-27T02:33:59.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:59 smithi151 ceph-mon[64897]: pgmap v201: 161 pgs: 3 active+undersized+degraded, 26 peering, 5 stale+active+clean, 7 active+undersized, 120 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 16/723 objects degraded (2.213%) 2024-04-27T02:33:59.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:59 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 16/723 objects degraded (2.213%), 3 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:59.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:59 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:59.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:59 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:59.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:33:59 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:59.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[82260]: pgmap v201: 161 pgs: 3 active+undersized+degraded, 26 peering, 5 stale+active+clean, 7 active+undersized, 120 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 16/723 objects degraded (2.213%) 2024-04-27T02:33:59.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 16/723 objects degraded (2.213%), 3 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:59.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:59.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:59.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:33:59.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[87214]: pgmap v201: 161 pgs: 3 active+undersized+degraded, 26 peering, 5 stale+active+clean, 7 active+undersized, 120 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 16/723 objects degraded (2.213%) 2024-04-27T02:33:59.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 16/723 objects degraded (2.213%), 3 pgs degraded (PG_DEGRADED) 2024-04-27T02:33:59.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:59.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:33:59.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:33:59 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:00.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:00 smithi151 ceph-mon[64897]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-04-27T02:34:00.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:00 smithi026 ceph-mon[82260]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-04-27T02:34:00.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:00 smithi026 ceph-mon[87214]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-04-27T02:34:01.555 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:01 smithi151 ceph-mon[64897]: pgmap v202: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-04-27T02:34:01.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:01 smithi026 ceph-mon[82260]: pgmap v202: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-04-27T02:34:01.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:01 smithi026 ceph-mon[87214]: pgmap v202: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-04-27T02:34:02.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:02 smithi151 ceph-mon[64897]: from='osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:34:02.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:02 smithi151 ceph-mon[64897]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:34:02.594 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:34:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:34:02.229+0000 7f6d3fbe77c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-27T02:34:02.594 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:34:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:34:02.255+0000 7f6d3fbe77c0 -1 osd.6 141 log_to_monitors true 2024-04-27T02:34:02.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:02 smithi026 ceph-mon[82260]: from='osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:34:02.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:02 smithi026 ceph-mon[82260]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:34:02.603 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:34:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:34:02] "GET /metrics HTTP/1.1" 200 38207 "" "Prometheus/2.43.0" 2024-04-27T02:34:02.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:02 smithi026 ceph-mon[87214]: from='osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:34:02.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:02 smithi026 ceph-mon[87214]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-27T02:34:03.093 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:34:02 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:34:02.788+0000 7f6d2d037700 -1 osd.6 141 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:34:03.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:03 smithi151 ceph-mon[64897]: pgmap v203: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-04-27T02:34:03.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:03 smithi151 ceph-mon[64897]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-27T02:34:03.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:03 smithi151 ceph-mon[64897]: osdmap e144: 8 total, 7 up, 8 in 2024-04-27T02:34:03.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:03 smithi151 ceph-mon[64897]: from='osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:03.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:03 smithi151 ceph-mon[64897]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:03.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:03 smithi151 ceph-mon[64897]: from='osd.6 ' entity='osd.6' 2024-04-27T02:34:03.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[82260]: pgmap v203: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-04-27T02:34:03.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[82260]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-27T02:34:03.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[82260]: osdmap e144: 8 total, 7 up, 8 in 2024-04-27T02:34:03.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[82260]: from='osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:03.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[82260]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:03.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[82260]: from='osd.6 ' entity='osd.6' 2024-04-27T02:34:03.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[87214]: pgmap v203: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-04-27T02:34:03.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[87214]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-27T02:34:03.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[87214]: osdmap e144: 8 total, 7 up, 8 in 2024-04-27T02:34:03.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[87214]: from='osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:03.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[87214]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:03.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:03 smithi026 ceph-mon[87214]: from='osd.6 ' entity='osd.6' 2024-04-27T02:34:04.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:04.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:04.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:04 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:34:04.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:04 smithi151 ceph-mon[64897]: osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858] boot 2024-04-27T02:34:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:04 smithi151 ceph-mon[64897]: osdmap e145: 8 total, 8 up, 8 in 2024-04-27T02:34:04.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:04 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:34:04.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:04.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:04.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:34:04.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[82260]: osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858] boot 2024-04-27T02:34:04.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[82260]: osdmap e145: 8 total, 8 up, 8 in 2024-04-27T02:34:04.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:34:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:34:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[87214]: osd.6 [v2:172.21.15.151:6816/2323277858,v1:172.21.15.151:6817/2323277858] boot 2024-04-27T02:34:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[87214]: osdmap e145: 8 total, 8 up, 8 in 2024-04-27T02:34:04.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:04 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-27T02:34:05.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:05 smithi151 ceph-mon[64897]: pgmap v206: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 82/723 objects degraded (11.342%) 2024-04-27T02:34:05.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:05.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:05 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:05.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:05 smithi151 ceph-mon[64897]: osdmap e146: 8 total, 8 up, 8 in 2024-04-27T02:34:05.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[82260]: pgmap v206: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 82/723 objects degraded (11.342%) 2024-04-27T02:34:05.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:05.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:05.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[82260]: osdmap e146: 8 total, 8 up, 8 in 2024-04-27T02:34:05.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[87214]: pgmap v206: 161 pgs: 16 active+undersized+degraded, 37 active+undersized, 108 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 82/723 objects degraded (11.342%) 2024-04-27T02:34:05.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:05.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:05.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:05 smithi026 ceph-mon[87214]: osdmap e146: 8 total, 8 up, 8 in 2024-04-27T02:34:06.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:06 smithi151 ceph-mon[64897]: pgmap v208: 161 pgs: 30 peering, 6 active+undersized+degraded, 17 active+undersized, 108 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 29/723 objects degraded (4.011%) 2024-04-27T02:34:06.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:06 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 29/723 objects degraded (4.011%), 6 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:07.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:06 smithi026 ceph-mon[87214]: pgmap v208: 161 pgs: 30 peering, 6 active+undersized+degraded, 17 active+undersized, 108 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 29/723 objects degraded (4.011%) 2024-04-27T02:34:07.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:06 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 29/723 objects degraded (4.011%), 6 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:07.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:06 smithi026 ceph-mon[82260]: pgmap v208: 161 pgs: 30 peering, 6 active+undersized+degraded, 17 active+undersized, 108 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 29/723 objects degraded (4.011%) 2024-04-27T02:34:07.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:06 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 29/723 objects degraded (4.011%), 6 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:08.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:07 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:08.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:07 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:08.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:07 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:08.852 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:34:08 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:34:08.379+0000 7fc7ce81b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-27T02:34:09.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: pgmap v209: 161 pgs: 30 peering, 1 active+undersized, 130 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/723 objects degraded (4.011%), 6 pgs degraded) 2024-04-27T02:34:09.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:09 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:34:09.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: pgmap v209: 161 pgs: 30 peering, 1 active+undersized, 130 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:34:09.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:09.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:09.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/723 objects degraded (4.011%), 6 pgs degraded) 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: pgmap v209: 161 pgs: 30 peering, 1 active+undersized, 130 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/723 objects degraded (4.011%), 6 pgs degraded) 2024-04-27T02:34:09.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:09 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:34:11.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:11 smithi151 ceph-mon[64897]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:34:11.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:11.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:11 smithi026 ceph-mon[82260]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:34:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:11.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:11.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:11 smithi026 ceph-mon[87214]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:34:11.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:11.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:11.827 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:34:12.506 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:34:12.506 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (8m) 112s ago 30m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:34:12.506 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (7m) 9s ago 29m 78.0M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (8m) 112s ago 29m 62.1M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (9m) 9s ago 34m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (19m) 112s ago 36m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) 112s ago 36m 50.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (6m) 9s ago 34m 48.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (5m) 112s ago 35m 41.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (8m) 112s ago 30m 16.3M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (8m) 9s ago 30m 15.8M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (4m) 112s ago 34m 50.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (3m) 112s ago 33m 47.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (2m) 112s ago 33m 41.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (117s) 112s ago 33m 12.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:34:12.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (82s) 9s ago 32m 52.4M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:34:12.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (47s) 9s ago 32m 50.5M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:34:12.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (13s) 9s ago 32m 14.9M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:34:12.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (31m) 9s ago 31m 61.1M 3248M 17.2.0 e1d6a67b021e aa10c241e9de 2024-04-27T02:34:12.508 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (8m) 9s ago 30m 70.9M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:34:12.508 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (29m) 112s ago 29m 78.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:34:12.508 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (29m) 9s ago 29m 81.1M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:34:12.508 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:34:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:34:12] "GET /metrics HTTP/1.1" 200 38213 "" "Prometheus/2.43.0" 2024-04-27T02:34:12.923 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:34:12.923 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:34:12.923 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:34:12.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:12.923 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 7 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:34:12.924 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:34:12.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:12.925 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:34:12.925 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-27T02:34:12.925 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 12 2024-04-27T02:34:12.925 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:34:12.925 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:34:13.270 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:34:13.270 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:34:13.270 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:34:13.270 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-27T02:34:13.270 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:34:13.271 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "7/8 daemons upgraded", 2024-04-27T02:34:13.271 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-27T02:34:13.271 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:34:13.271 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:34:13.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:13 smithi151 ceph-mon[64897]: from='client.44538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:13 smithi151 ceph-mon[64897]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-27T02:34:13.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:13 smithi151 ceph-mon[64897]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:13 smithi151 ceph-mon[64897]: from='client.54487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:13 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/3783177338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:13.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[82260]: from='client.44538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[82260]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-27T02:34:13.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[82260]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[82260]: from='client.54487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/3783177338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:13.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[87214]: from='client.44538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[87214]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-27T02:34:13.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[87214]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[87214]: from='client.54487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:13.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:13 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/3783177338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:14.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:14 smithi151 ceph-mon[64897]: from='client.44562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:14.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:14 smithi026 ceph-mon[82260]: from='client.44562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:14.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:14 smithi026 ceph-mon[87214]: from='client.44562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:15.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:15 smithi151 ceph-mon[64897]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:34:15.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:15 smithi026 ceph-mon[82260]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:34:15.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:15 smithi026 ceph-mon[87214]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:34:17.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:17 smithi026 ceph-mon[82260]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-04-27T02:34:17.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:17 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:17 smithi026 ceph-mon[87214]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-04-27T02:34:17.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:17 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:17.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:17 smithi151 ceph-mon[64897]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-04-27T02:34:17.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:17 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:18.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:18 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:18.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:18 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:18.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:18 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:19.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:19 smithi151 ceph-mon[64897]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:19.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:19 smithi026 ceph-mon[82260]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:19.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:19 smithi026 ceph-mon[87214]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:21.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:21 smithi151 ceph-mon[64897]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:34:21.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:21 smithi026 ceph-mon[82260]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:34:21.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:21 smithi026 ceph-mon[87214]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:34:22.503 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:34:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:34:22] "GET /metrics HTTP/1.1" 200 38213 "" "Prometheus/2.43.0" 2024-04-27T02:34:23.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:23 smithi151 ceph-mon[64897]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:23.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:23 smithi026 ceph-mon[82260]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:23.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:23 smithi026 ceph-mon[87214]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:24.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:24 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:24.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:24 smithi151 ceph-mon[64897]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:24.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:24 smithi151 ceph-mon[64897]: Upgrade: osd.7 is safe to restart 2024-04-27T02:34:24.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:24 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:24.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:24 smithi026 ceph-mon[82260]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:24.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:24 smithi026 ceph-mon[82260]: Upgrade: osd.7 is safe to restart 2024-04-27T02:34:24.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:24 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:24.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:24 smithi026 ceph-mon[87214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-27T02:34:24.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:24 smithi026 ceph-mon[87214]: Upgrade: osd.7 is safe to restart 2024-04-27T02:34:25.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:25 smithi151 ceph-mon[64897]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:25.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:25.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:34:25.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:25.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[82260]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:25.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:25.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:34:25.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:25.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[87214]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:25.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:25.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-27T02:34:25.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:26.583 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:26 smithi151 ceph-mon[64897]: Upgrade: Updating osd.7 2024-04-27T02:34:26.583 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:26 smithi151 ceph-mon[64897]: Deploying daemon osd.7 on smithi151 2024-04-27T02:34:26.583 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:26.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:26 smithi026 ceph-mon[82260]: Upgrade: Updating osd.7 2024-04-27T02:34:26.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:26 smithi026 ceph-mon[82260]: Deploying daemon osd.7 on smithi151 2024-04-27T02:34:26.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:26.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:26 smithi026 ceph-mon[87214]: Upgrade: Updating osd.7 2024-04-27T02:34:26.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:26 smithi026 ceph-mon[87214]: Deploying daemon osd.7 on smithi151 2024-04-27T02:34:26.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:27.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:27 smithi026 ceph-mon[82260]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:34:27.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:27 smithi026 ceph-mon[87214]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:34:27.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:27 smithi151 ceph-mon[64897]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:34:28.503 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:28 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:28.503 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:28 smithi151 systemd[1]: Stopping Ceph osd.7 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:34:28.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:28 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:28.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:28 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:28.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[34804]: 2024-04-27T02:34:28.528+0000 7ff10d9c6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:34:28.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[34804]: 2024-04-27T02:34:28.528+0000 7ff10d9c6700 -1 osd.7 146 *** Got signal Terminated *** 2024-04-27T02:34:28.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[34804]: 2024-04-27T02:34:28.528+0000 7ff10d9c6700 -1 osd.7 146 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:34:29.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:29 smithi151 ceph-mon[64897]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:29.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:29 smithi151 ceph-mon[64897]: osd.7 marked itself down and dead 2024-04-27T02:34:29.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:29 smithi026 ceph-mon[87214]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:29.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:29 smithi026 ceph-mon[87214]: osd.7 marked itself down and dead 2024-04-27T02:34:29.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:29 smithi026 ceph-mon[82260]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:34:29.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:29 smithi026 ceph-mon[82260]: osd.7 marked itself down and dead 2024-04-27T02:34:30.093 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:29 smithi151 podman[89723]: 2024-04-27 02:34:29.807096024 +0000 UTC m=+1.399974263 container died aa10c241e9de1833e6795ea084a1c457cfbbead7aeaab6589c92656792326cff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, release=754, io.openshift.tags=base centos centos-stream, name=centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870) 2024-04-27T02:34:30.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:30 smithi151 ceph-mon[64897]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:34:30.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:30 smithi151 ceph-mon[64897]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-27T02:34:30.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:30 smithi151 ceph-mon[64897]: osdmap e147: 8 total, 7 up, 8 in 2024-04-27T02:34:30.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:30 smithi151 podman[89723]: 2024-04-27 02:34:30.128260091 +0000 UTC m=+1.721138333 container cleanup aa10c241e9de1833e6795ea084a1c457cfbbead7aeaab6589c92656792326cff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, ceph=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-type=git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-27T02:34:30.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:30 smithi151 bash[89723]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7 2024-04-27T02:34:30.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:30 smithi151 podman[89742]: 2024-04-27 02:34:30.338014912 +0000 UTC m=+0.529576826 container remove aa10c241e9de1833e6795ea084a1c457cfbbead7aeaab6589c92656792326cff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.expose-services=, com.redhat.component=centos-stream-container, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-27T02:34:30.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:30 smithi026 ceph-mon[82260]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:34:30.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:30 smithi026 ceph-mon[82260]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-27T02:34:30.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:30 smithi026 ceph-mon[82260]: osdmap e147: 8 total, 7 up, 8 in 2024-04-27T02:34:30.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:30 smithi026 ceph-mon[87214]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-27T02:34:30.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:30 smithi026 ceph-mon[87214]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-27T02:34:30.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:30 smithi026 ceph-mon[87214]: osdmap e147: 8 total, 7 up, 8 in 2024-04-27T02:34:31.056 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:30 smithi151 podman[89835]: 2024-04-27 02:34:30.65868346 +0000 UTC m=+0.018409585 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:34:31.056 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:30 smithi151 podman[89835]: 2024-04-27 02:34:30.755226615 +0000 UTC m=+0.114952730 container create 497b46193962b5277e2cbfd206f487bdbcddf84d4a09e4f3dff9e38645882e5f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-04-27T02:34:31.056 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:30 smithi151 podman[89835]: 2024-04-27 02:34:30.999418692 +0000 UTC m=+0.359144808 container init 497b46193962b5277e2cbfd206f487bdbcddf84d4a09e4f3dff9e38645882e5f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:34:31.056 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:31 smithi151 podman[89835]: 2024-04-27 02:34:31.002707047 +0000 UTC m=+0.362433173 container start 497b46193962b5277e2cbfd206f487bdbcddf84d4a09e4f3dff9e38645882e5f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:34:31.056 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:31 smithi151 podman[89835]: 2024-04-27 02:34:31.082583884 +0000 UTC m=+0.442310003 container attach 497b46193962b5277e2cbfd206f487bdbcddf84d4a09e4f3dff9e38645882e5f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-27T02:34:31.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:31 smithi151 ceph-mon[64897]: pgmap v221: 161 pgs: 30 peering, 12 stale+active+clean, 119 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:34:31.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:31 smithi151 ceph-mon[64897]: Health check failed: Reduced data availability: 3 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-27T02:34:31.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:31 smithi151 ceph-mon[64897]: osdmap e148: 8 total, 7 up, 8 in 2024-04-27T02:34:31.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:31 smithi151 podman[89835]: 2024-04-27 02:34:31.292606899 +0000 UTC m=+0.652333015 container died 497b46193962b5277e2cbfd206f487bdbcddf84d4a09e4f3dff9e38645882e5f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:34:31.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:31 smithi026 ceph-mon[87214]: pgmap v221: 161 pgs: 30 peering, 12 stale+active+clean, 119 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:34:31.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:31 smithi026 ceph-mon[87214]: Health check failed: Reduced data availability: 3 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-27T02:34:31.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:31 smithi026 ceph-mon[87214]: osdmap e148: 8 total, 7 up, 8 in 2024-04-27T02:34:31.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:31 smithi026 ceph-mon[82260]: pgmap v221: 161 pgs: 30 peering, 12 stale+active+clean, 119 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-27T02:34:31.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:31 smithi026 ceph-mon[82260]: Health check failed: Reduced data availability: 3 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-27T02:34:31.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:31 smithi026 ceph-mon[82260]: osdmap e148: 8 total, 7 up, 8 in 2024-04-27T02:34:31.867 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:31 smithi151 podman[89861]: 2024-04-27 02:34:31.869548865 +0000 UTC m=+0.571627719 container remove 497b46193962b5277e2cbfd206f487bdbcddf84d4a09e4f3dff9e38645882e5f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-27T02:34:31.868 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:31 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-27T02:34:31.868 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:31 smithi151 systemd[1]: Stopped Ceph osd.7 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:34:31.868 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:31 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.7.service: Consumed 17.016s CPU time. 2024-04-27T02:34:32.281 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:32 smithi151 systemd[1]: Starting Ceph osd.7 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:34:32.555 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:32 smithi151 podman[89965]: 2024-04-27 02:34:32.210834148 +0000 UTC m=+0.018935963 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:34:32.555 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:32 smithi151 podman[89965]: 2024-04-27 02:34:32.349035232 +0000 UTC m=+0.157137022 container create c967c539f06db6d08c786fc4133af7d8cfd3c38006cd08018f3cb3215c141e3d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:34:32.555 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:32 smithi151 podman[89965]: 2024-04-27 02:34:32.581641872 +0000 UTC m=+0.389743673 container init c967c539f06db6d08c786fc4133af7d8cfd3c38006cd08018f3cb3215c141e3d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-27T02:34:32.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:34:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:34:32] "GET /metrics HTTP/1.1" 200 38217 "" "Prometheus/2.43.0" 2024-04-27T02:34:32.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:32 smithi151 podman[89965]: 2024-04-27 02:34:32.58556721 +0000 UTC m=+0.393669001 container start c967c539f06db6d08c786fc4133af7d8cfd3c38006cd08018f3cb3215c141e3d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:34:32.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:32 smithi151 podman[89965]: 2024-04-27 02:34:32.664529335 +0000 UTC m=+0.472631128 container attach c967c539f06db6d08c786fc4133af7d8cfd3c38006cd08018f3cb3215c141e3d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-27T02:34:33.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-mon[64897]: pgmap v223: 161 pgs: 30 peering, 12 stale+active+clean, 119 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:34:33.343 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate[89979]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-27T02:34:33.343 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate[89979]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 bash[89965]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 bash[89965]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 bash[89965]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 bash[89965]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 bash[89965]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 bash[89965]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 bash[89965]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate[89979]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate[89979]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate[89979]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate[89979]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate[89979]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-27T02:34:33.344 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 podman[89965]: 2024-04-27 02:34:33.049771291 +0000 UTC m=+0.857873098 container died c967c539f06db6d08c786fc4133af7d8cfd3c38006cd08018f3cb3215c141e3d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-27T02:34:33.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:33 smithi026 ceph-mon[82260]: pgmap v223: 161 pgs: 30 peering, 12 stale+active+clean, 119 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:34:33.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:33 smithi026 ceph-mon[87214]: pgmap v223: 161 pgs: 30 peering, 12 stale+active+clean, 119 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:34:33.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 podman[90108]: 2024-04-27 02:34:33.751415674 +0000 UTC m=+0.696179114 container remove c967c539f06db6d08c786fc4133af7d8cfd3c38006cd08018f3cb3215c141e3d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-activate, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-27T02:34:34.227 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 podman[90175]: 2024-04-27 02:34:33.952364535 +0000 UTC m=+0.099699503 container create bec04d87416c4d23463803ae25bceb5650b301ffa2656d802f92371b8e1e48ef (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-27T02:34:34.228 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:33 smithi151 podman[90175]: 2024-04-27 02:34:33.871070146 +0000 UTC m=+0.018405076 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:34:34.228 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:34 smithi151 podman[90175]: 2024-04-27 02:34:34.154425719 +0000 UTC m=+0.301760631 container init bec04d87416c4d23463803ae25bceb5650b301ffa2656d802f92371b8e1e48ef (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-27T02:34:34.228 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:34 smithi151 podman[90175]: 2024-04-27 02:34:34.15743481 +0000 UTC m=+0.304769725 container start bec04d87416c4d23463803ae25bceb5650b301ffa2656d802f92371b8e1e48ef (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:34:34.228 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:34 smithi151 ceph-osd[90193]: -- 172.21.15.151:0/2000670970 <== mon.2 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ec0b255680 con 0x55ec0b21ec00 2024-04-27T02:34:34.228 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:34 smithi151 bash[90175]: bec04d87416c4d23463803ae25bceb5650b301ffa2656d802f92371b8e1e48ef 2024-04-27T02:34:34.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:34 smithi151 systemd[1]: Started Ceph osd.7 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:34:34.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:34 smithi151 ceph-mon[64897]: Health check failed: Degraded data redundancy: 13/723 objects degraded (1.798%), 5 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:34.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:34.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:34.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:34 smithi026 ceph-mon[87214]: Health check failed: Degraded data redundancy: 13/723 objects degraded (1.798%), 5 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:34.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:34.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:34.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:34 smithi026 ceph-mon[82260]: Health check failed: Degraded data redundancy: 13/723 objects degraded (1.798%), 5 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:34.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:34.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:35.093 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:34:34.701+0000 7f4f974027c0 -1 Falling back to public interface 2024-04-27T02:34:35.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:35 smithi151 ceph-mon[64897]: pgmap v224: 161 pgs: 5 active+undersized+degraded, 30 peering, 4 stale+active+clean, 19 active+undersized, 103 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 13/723 objects degraded (1.798%) 2024-04-27T02:34:35.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:35 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:35.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:35 smithi026 ceph-mon[82260]: pgmap v224: 161 pgs: 5 active+undersized+degraded, 30 peering, 4 stale+active+clean, 19 active+undersized, 103 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 13/723 objects degraded (1.798%) 2024-04-27T02:34:35.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:35 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:35.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:35 smithi026 ceph-mon[87214]: pgmap v224: 161 pgs: 5 active+undersized+degraded, 30 peering, 4 stale+active+clean, 19 active+undersized, 103 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 13/723 objects degraded (1.798%) 2024-04-27T02:34:35.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:35 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:36.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:36 smithi026 ceph-mon[82260]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 10 pgs peering) 2024-04-27T02:34:36.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:36 smithi026 ceph-mon[87214]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 10 pgs peering) 2024-04-27T02:34:36.728 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:36 smithi151 ceph-mon[64897]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 10 pgs peering) 2024-04-27T02:34:37.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:37 smithi026 ceph-mon[82260]: pgmap v225: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:37.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:37 smithi026 ceph-mon[87214]: pgmap v225: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:37.630 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:37 smithi151 ceph-mon[64897]: pgmap v225: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:37.630 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:34:37.416+0000 7f4f974027c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-27T02:34:37.630 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:34:37.449+0000 7f4f974027c0 -1 osd.7 146 log_to_monitors true 2024-04-27T02:34:38.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:38 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:38.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:38 smithi151 ceph-mon[64897]: from='osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:34:38.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:38 smithi151 ceph-mon[64897]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:34:38.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:38 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:38.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:38 smithi026 ceph-mon[82260]: from='osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:34:38.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:38 smithi026 ceph-mon[82260]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:34:38.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:38 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:38 smithi026 ceph-mon[87214]: from='osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:34:38.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:38 smithi026 ceph-mon[87214]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-27T02:34:39.093 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:34:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:34:38.856+0000 7f4f84852700 -1 osd.7 146 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-27T02:34:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: pgmap v226: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-27T02:34:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: osdmap e149: 8 total, 7 up, 8 in 2024-04-27T02:34:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: from='osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:39.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[82260]: from='osd.7 ' entity='osd.7' 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: pgmap v226: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: osdmap e149: 8 total, 7 up, 8 in 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: from='osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:39.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:39 smithi026 ceph-mon[87214]: from='osd.7 ' entity='osd.7' 2024-04-27T02:34:39.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: pgmap v226: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:39.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-27T02:34:39.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: osdmap e149: 8 total, 7 up, 8 in 2024-04-27T02:34:39.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: from='osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:39.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-27T02:34:39.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:39.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:39.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:39 smithi151 ceph-mon[64897]: from='osd.7 ' entity='osd.7' 2024-04-27T02:34:41.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:41.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:41.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:40 smithi151 ceph-mon[64897]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:34:41.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:40 smithi151 ceph-mon[64897]: osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642] boot 2024-04-27T02:34:41.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:40 smithi151 ceph-mon[64897]: osdmap e150: 8 total, 8 up, 8 in 2024-04-27T02:34:41.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:34:41.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:40 smithi151 ceph-mon[64897]: pgmap v229: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:41.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:41.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:41.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[82260]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:34:41.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[82260]: osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642] boot 2024-04-27T02:34:41.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[82260]: osdmap e150: 8 total, 8 up, 8 in 2024-04-27T02:34:41.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:34:41.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[82260]: pgmap v229: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[87214]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-27T02:34:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[87214]: osd.7 [v2:172.21.15.151:6824/3492417642,v1:172.21.15.151:6825/3492417642] boot 2024-04-27T02:34:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[87214]: osdmap e150: 8 total, 8 up, 8 in 2024-04-27T02:34:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-27T02:34:41.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:40 smithi026 ceph-mon[87214]: pgmap v229: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-04-27T02:34:42.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:42.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:42.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:41 smithi151 ceph-mon[64897]: osdmap e151: 8 total, 8 up, 8 in 2024-04-27T02:34:42.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:42.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:42.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:41 smithi026 ceph-mon[87214]: osdmap e151: 8 total, 8 up, 8 in 2024-04-27T02:34:42.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:42.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:42.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:41 smithi026 ceph-mon[82260]: osdmap e151: 8 total, 8 up, 8 in 2024-04-27T02:34:42.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:34:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:34:42] "GET /metrics HTTP/1.1" 200 38168 "" "Prometheus/2.43.0" 2024-04-27T02:34:43.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:42 smithi151 ceph-mon[64897]: pgmap v231: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 98/723 objects degraded (13.555%) 2024-04-27T02:34:43.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:42 smithi151 ceph-mon[64897]: Health check update: Degraded data redundancy: 98/723 objects degraded (13.555%), 25 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:43.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:42 smithi026 ceph-mon[82260]: pgmap v231: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 98/723 objects degraded (13.555%) 2024-04-27T02:34:43.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:42 smithi026 ceph-mon[82260]: Health check update: Degraded data redundancy: 98/723 objects degraded (13.555%), 25 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:43.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:42 smithi026 ceph-mon[87214]: pgmap v231: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 98/723 objects degraded (13.555%) 2024-04-27T02:34:43.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:42 smithi026 ceph-mon[87214]: Health check update: Degraded data redundancy: 98/723 objects degraded (13.555%), 25 pgs degraded (PG_DEGRADED) 2024-04-27T02:34:43.648 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:34:44.329 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (8m) 2m ago 30m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (8m) 5s ago 30m 80.6M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (9m) 2m ago 29m 62.1M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (9m) 5s ago 35m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (19m) 2m ago 36m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) 2m ago 36m 50.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (7m) 5s ago 35m 48.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (6m) 2m ago 35m 41.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (9m) 2m ago 31m 16.3M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (9m) 5s ago 31m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (4m) 2m ago 34m 50.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:34:44.330 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (4m) 2m ago 34m 47.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (3m) 2m ago 34m 41.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (2m) 2m ago 33m 12.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (114s) 5s ago 33m 54.0M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (79s) 5s ago 32m 52.2M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (45s) 5s ago 32m 53.3M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (10s) 5s ago 32m 12.5M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 bec04d87416c 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (8m) 5s ago 31m 70.9M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (29m) 2m ago 29m 78.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:34:44.331 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (29m) 5s ago 29m 81.3M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:34:44.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all osd 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-27T02:34:44.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: Upgrade: Setting require_osd_release to 19 squid 2024-04-27T02:34:44.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-27T02:34:44.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:44.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all osd 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-27T02:34:44.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-27T02:34:44.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all osd 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-27T02:34:44.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-27T02:34:44.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: Upgrade: Setting require_osd_release to 19 squid 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: Upgrade: Setting require_osd_release to 19 squid 2024-04-27T02:34:44.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-27T02:34:44.806 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 8 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:34:44.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:34:44.808 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:34:44.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-27T02:34:44.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 13 2024-04-27T02:34:44.808 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:34:44.808 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:34:45.152 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='client.54508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: pgmap v232: 161 pgs: 13 active+undersized+degraded, 17 active+undersized, 131 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 45/723 objects degraded (6.224%) 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: osdmap e152: 8 total, 8 up, 8 in 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mds 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nfs 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Upgrade: Finalizing container_image settings 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-27T02:34:45.596 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: Upgrade: Complete! 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:45.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:45 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/3566046563' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='client.54508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: pgmap v232: 161 pgs: 13 active+undersized+degraded, 17 active+undersized, 131 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 45/723 objects degraded (6.224%) 2024-04-27T02:34:45.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: osdmap e152: 8 total, 8 up, 8 in 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mds 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nfs 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Upgrade: Finalizing container_image settings 2024-04-27T02:34:45.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='client.54508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: pgmap v232: 161 pgs: 13 active+undersized+degraded, 17 active+undersized, 131 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 45/723 objects degraded (6.224%) 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: osdmap e152: 8 total, 8 up, 8 in 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mds 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nfs 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Upgrade: Finalizing container_image settings 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:34:45.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:34:45.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:34:45.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: Upgrade: Complete! 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/3566046563' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:45.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: Upgrade: Complete! 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:34:45.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:45.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:45 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/3566046563' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:34:46.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:46 smithi151 ceph-mon[64897]: from='client.44595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:46.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:46 smithi026 ceph-mon[82260]: from='client.44595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:46.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:46 smithi026 ceph-mon[87214]: from='client.44595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:34:47.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[82260]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-04-27T02:34:47.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[82260]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/723 objects degraded (6.224%), 13 pgs degraded) 2024-04-27T02:34:47.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[82260]: Cluster is now healthy 2024-04-27T02:34:47.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:47.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[87214]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-04-27T02:34:47.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[87214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/723 objects degraded (6.224%), 13 pgs degraded) 2024-04-27T02:34:47.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[87214]: Cluster is now healthy 2024-04-27T02:34:47.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:47 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:47.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:47 smithi151 ceph-mon[64897]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-04-27T02:34:47.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:47 smithi151 ceph-mon[64897]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/723 objects degraded (6.224%), 13 pgs degraded) 2024-04-27T02:34:47.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:47 smithi151 ceph-mon[64897]: Cluster is now healthy 2024-04-27T02:34:47.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:47 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:48.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:48.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:48.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:49.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:49 smithi151 ceph-mon[64897]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:34:49.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:49 smithi026 ceph-mon[82260]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:34:49.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:49 smithi026 ceph-mon[87214]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:34:51.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:51 smithi151 ceph-mon[64897]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:34:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:51 smithi026 ceph-mon[82260]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:34:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:51 smithi026 ceph-mon[87214]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-27T02:34:52.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:52 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:52.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:52 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:52.504 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:34:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:34:52] "GET /metrics HTTP/1.1" 200 38168 "" "Prometheus/2.43.0" 2024-04-27T02:34:52.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:52 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:53.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:53 smithi151 ceph-mon[64897]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:34:53.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:53 smithi026 ceph-mon[82260]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:34:53.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:53 smithi026 ceph-mon[87214]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:34:55.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:55 smithi151 ceph-mon[64897]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 32 op/s 2024-04-27T02:34:55.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:55 smithi026 ceph-mon[82260]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 32 op/s 2024-04-27T02:34:55.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:55 smithi026 ceph-mon[87214]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 32 op/s 2024-04-27T02:34:57.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:57.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:57.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:56 smithi151 ceph-mon[64897]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-27T02:34:57.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:57.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:57.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:56 smithi026 ceph-mon[82260]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-27T02:34:57.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:34:57.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:34:57.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:56 smithi026 ceph-mon[87214]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-27T02:34:58.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:57 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:58.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:57 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:58.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:57 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:34:59.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:34:58 smithi151 ceph-mon[64897]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-27T02:34:59.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:34:58 smithi026 ceph-mon[82260]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-27T02:34:59.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:34:58 smithi026 ceph-mon[87214]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-27T02:35:01.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:00 smithi151 ceph-mon[64897]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-27T02:35:01.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:00 smithi026 ceph-mon[82260]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-27T02:35:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:00 smithi026 ceph-mon[87214]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-27T02:35:02.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:35:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:35:02] "GET /metrics HTTP/1.1" 200 38305 "" "Prometheus/2.43.0" 2024-04-27T02:35:03.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:02 smithi151 ceph-mon[64897]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-27T02:35:03.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:02 smithi026 ceph-mon[82260]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-27T02:35:03.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:02 smithi026 ceph-mon[87214]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-27T02:35:05.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:04 smithi151 ceph-mon[64897]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-27T02:35:05.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:04 smithi026 ceph-mon[82260]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-27T02:35:05.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:04 smithi026 ceph-mon[87214]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-27T02:35:07.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:06 smithi151 ceph-mon[64897]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 0 B/s wr, 93 op/s 2024-04-27T02:35:07.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:06 smithi026 ceph-mon[82260]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 0 B/s wr, 93 op/s 2024-04-27T02:35:07.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:06 smithi026 ceph-mon[87214]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 0 B/s wr, 93 op/s 2024-04-27T02:35:08.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:07 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:08.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:07 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:08.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:07 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:09.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:08 smithi151 ceph-mon[64897]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 27 op/s 2024-04-27T02:35:09.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:08 smithi026 ceph-mon[82260]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 27 op/s 2024-04-27T02:35:09.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:08 smithi026 ceph-mon[87214]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 27 op/s 2024-04-27T02:35:11.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:10 smithi151 ceph-mon[64897]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 24 op/s 2024-04-27T02:35:11.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:10 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:11.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:10 smithi026 ceph-mon[82260]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 24 op/s 2024-04-27T02:35:11.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:10 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:11.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:10 smithi026 ceph-mon[87214]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 24 op/s 2024-04-27T02:35:11.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:10 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:12.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:35:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:35:12] "GET /metrics HTTP/1.1" 200 38304 "" "Prometheus/2.43.0" 2024-04-27T02:35:13.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:12 smithi151 ceph-mon[64897]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:13.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:12 smithi026 ceph-mon[82260]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:13.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:12 smithi026 ceph-mon[87214]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:15.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:14 smithi151 ceph-mon[64897]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:15.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:14 smithi026 ceph-mon[82260]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:15.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:14 smithi026 ceph-mon[87214]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:16.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:15 smithi026 ceph-mon[82260]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:16.267 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:15 smithi026 ceph-mon[87214]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:16.296 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:35:16.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:15 smithi151 ceph-mon[64897]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:17.282 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:16 smithi026 ceph-mon[82260]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:17.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:16 smithi026 ceph-mon[87214]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:17.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:16 smithi151 ceph-mon[64897]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:18.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:17 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:18.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:17 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:18.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:17 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (9m) 2m ago 31m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (8m) 39s ago 30m 80.6M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (9m) 2m ago 30m 62.1M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (10m) 39s ago 35m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (20m) 2m ago 37m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (7m) 2m ago 37m 50.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (7m) 39s ago 35m 48.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (6m) 2m ago 36m 41.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (9m) 2m ago 31m 16.3M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:35:18.433 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (9m) 39s ago 31m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (5m) 2m ago 35m 50.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (5m) 2m ago 35m 47.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (3m) 2m ago 34m 41.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 2m ago 34m 12.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (2m) 39s ago 33m 54.0M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (113s) 39s ago 33m 52.2M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (79s) 39s ago 33m 53.3M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (44s) 39s ago 32m 12.5M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 bec04d87416c 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (9m) 39s ago 31m 70.9M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (30m) 2m ago 30m 78.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:35:18.434 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (30m) 39s ago 30m 81.3M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:35:19.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:18 smithi026 ceph-mon[82260]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:19.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:18 smithi026 ceph-mon[82260]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:19.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:18 smithi026 ceph-mon[87214]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:19.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:18 smithi026 ceph-mon[87214]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:19.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-04-27T02:35:19.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:18 smithi151 ceph-mon[64897]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:19.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:18 smithi151 ceph-mon[64897]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:35:21.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:20 smithi026 ceph-mon[82260]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:21.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:20 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/820793871' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:21.222 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:20 smithi026 ceph-mon[87214]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:21.222 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:20 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/820793871' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:21.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:20 smithi151 ceph-mon[64897]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:21.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:20 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/820793871' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:21.635 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-04-27T02:35:22.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:35:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:35:22] "GET /metrics HTTP/1.1" 200 38304 "" "Prometheus/2.43.0" 2024-04-27T02:35:23.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:22 smithi026 ceph-mon[82260]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:23.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:22 smithi026 ceph-mon[87214]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:23.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:22 smithi151 ceph-mon[64897]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:23.694 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)" 2024-04-27T02:35:24.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:23 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/307852868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:24.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:23 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/307852868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:24.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:35:24.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:23 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/307852868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:25.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:24 smithi026 ceph-mon[87214]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:25.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:24 smithi026 ceph-mon[82260]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:25.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:24 smithi151 ceph-mon[64897]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:26.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:26.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:35:26.303 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:35:26.304 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:35:26.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:26.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:35:27.244 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:26 smithi151 ceph-mon[64897]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:27.245 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:26 smithi151 ceph-mon[64897]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:27.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:26 smithi026 ceph-mon[82260]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:27.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:26 smithi026 ceph-mon[82260]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:27.272 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:26 smithi026 ceph-mon[87214]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:27.272 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:26 smithi026 ceph-mon[87214]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:28.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:27 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:28.267 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:27 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:28.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:27 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:28.914 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:35:29.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:29 smithi026 ceph-mon[82260]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:29.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:29 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/706338006' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:35:29.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:29 smithi026 ceph-mon[87214]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:29.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:29 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/706338006' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:35:29.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:29 smithi151 ceph-mon[64897]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:29.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:29 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/706338006' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:35:29.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-04-27T02:35:31.003 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:31 smithi026 ceph-mon[82260]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:31.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:31 smithi026 ceph-mon[87214]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:31.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:31 smithi151 ceph-mon[64897]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:32.346 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:32 smithi151 ceph-mon[64897]: from='client.54574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:32.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:32 smithi026 ceph-mon[87214]: from='client.54574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:32.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:32 smithi026 ceph-mon[82260]: from='client.54574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:32.352 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:35:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:35:32] "GET /metrics HTTP/1.1" 200 38305 "" "Prometheus/2.43.0" 2024-04-27T02:35:33.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:33 smithi026 ceph-mon[82260]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:33.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:33 smithi026 ceph-mon[87214]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:33.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:33 smithi151 ceph-mon[64897]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:33.604 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:35:34.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-27T02:35:34.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[82260]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:35:34.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:34.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:34.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:34.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:34.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:34.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[87214]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:35:34.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:34.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:34.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:34.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:34.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:34 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:35.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:34 smithi151 ceph-mon[64897]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:35:35.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:35.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:35.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:35.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:35.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:34 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:35.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:35 smithi026 ceph-mon[82260]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:35:35.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:35 smithi026 ceph-mon[82260]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:35.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:35 smithi026 ceph-mon[87214]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:35:35.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:35 smithi026 ceph-mon[87214]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:36.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:35 smithi151 ceph-mon[64897]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:35:36.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:35 smithi151 ceph-mon[64897]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:35:36.411 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:35:37.109 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (9m) 3m ago 31m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (9m) 58s ago 31m 80.6M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (10m) 3m ago 30m 62.1M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (10m) 58s ago 36m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (20m) 3m ago 37m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (7m) 3m ago 37m 50.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (8m) 58s ago 36m 48.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (6m) 3m ago 36m 41.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (10m) 3m ago 32m 16.3M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (10m) 58s ago 32m 15.5M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (5m) 3m ago 35m 50.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (5m) 3m ago 35m 47.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:35:37.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (4m) 3m ago 34m 41.6M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 3m ago 34m 12.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (2m) 58s ago 34m 54.0M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (2m) 58s ago 33m 52.2M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (98s) 58s ago 33m 53.3M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (63s) 58s ago 33m 12.5M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 bec04d87416c 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (9m) 58s ago 32m 70.9M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (30m) 3m ago 30m 78.5M - 17.2.0 e1d6a67b021e 5df1ba93687c 2024-04-27T02:35:37.111 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (30m) 58s ago 30m 81.3M - 17.2.0 e1d6a67b021e 334f798d9c6a 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 8 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:35:37.538 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-27T02:35:37.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:35:37.539 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:35:37.539 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-27T02:35:37.539 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 13 2024-04-27T02:35:37.539 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:35:37.539 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:35:37.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:37.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='client.54580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:37.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/1787444538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='client.54580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:37.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:37 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/1787444538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:37.899 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/2 daemons upgraded", 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading rgw daemons", 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:35:37.900 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:35:38.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:38.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='client.54580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:38.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:38.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:38.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:38.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:38.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:38.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:37 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/1787444538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: from='client.44655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mgr 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mon 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all crash 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all osd 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mds 2024-04-27T02:35:38.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: from='client.44655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mgr 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mon 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all crash 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all osd 2024-04-27T02:35:38.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:38 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mds 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: from='client.44655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mgr 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mon 2024-04-27T02:35:39.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all crash 2024-04-27T02:35:39.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all osd 2024-04-27T02:35:39.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:38 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mds 2024-04-27T02:35:39.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[87214]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:39.787 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[87214]: from='client.44667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[82260]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[82260]: from='client.44667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:35:39.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:39 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:40.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:39 smithi151 ceph-mon[64897]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:40.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:39 smithi151 ceph-mon[64897]: from='client.44667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:35:40.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:39 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:40.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:39 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.oulbnt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:35:40.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:39 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:40.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:40 smithi026 ceph-mon[82260]: Upgrade: Updating rgw.foo.smithi026.oulbnt (1/2) 2024-04-27T02:35:40.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:40 smithi026 ceph-mon[82260]: Deploying daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:35:40.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:40 smithi026 ceph-mon[82260]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:40.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:40 smithi026 ceph-mon[87214]: Upgrade: Updating rgw.foo.smithi026.oulbnt (1/2) 2024-04-27T02:35:40.644 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:40 smithi026 ceph-mon[87214]: Deploying daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:35:40.644 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:40 smithi026 ceph-mon[87214]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:41.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:40 smithi151 ceph-mon[64897]: Upgrade: Updating rgw.foo.smithi026.oulbnt (1/2) 2024-04-27T02:35:41.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:40 smithi151 ceph-mon[64897]: Deploying daemon rgw.foo.smithi026.oulbnt on smithi026 2024-04-27T02:35:41.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:40 smithi151 ceph-mon[64897]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:35:41.730 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:41.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:42.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:42.345 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:35:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:35:42] "GET /metrics HTTP/1.1" 200 38304 "" "Prometheus/2.43.0" 2024-04-27T02:35:43.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:42 smithi151 ceph-mon[64897]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:43.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:42 smithi026 ceph-mon[82260]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:43.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:42 smithi026 ceph-mon[87214]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:35:44.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:43 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:44.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:43 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:44.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:43 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:44.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:43 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:44.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:43 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:44.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:43 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:45.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:45 smithi151 ceph-mon[64897]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:35:45.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:45 smithi151 ceph-mon[64897]: Upgrade: Updating rgw.foo.smithi151.blhads (2/2) 2024-04-27T02:35:45.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:45.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:35:45.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:45.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:45 smithi151 ceph-mon[64897]: Deploying daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:35:45.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[87214]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:35:45.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[87214]: Upgrade: Updating rgw.foo.smithi151.blhads (2/2) 2024-04-27T02:35:45.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:45.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:35:45.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:45.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[87214]: Deploying daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:35:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[82260]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:35:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[82260]: Upgrade: Updating rgw.foo.smithi151.blhads (2/2) 2024-04-27T02:35:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi151.blhads", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-27T02:35:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:45.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:45 smithi026 ceph-mon[82260]: Deploying daemon rgw.foo.smithi151.blhads on smithi151 2024-04-27T02:35:47.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:47 smithi151 ceph-mon[64897]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 85 B/s wr, 113 op/s 2024-04-27T02:35:47.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:47 smithi026 ceph-mon[82260]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 85 B/s wr, 113 op/s 2024-04-27T02:35:47.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:47 smithi026 ceph-mon[87214]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 85 B/s wr, 113 op/s 2024-04-27T02:35:48.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:48 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:48.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:48 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:48.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:48 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:48.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:48 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:48.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:48.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:48.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:48.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:48.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:48.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:48.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:48.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:48 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:49.583 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:49 smithi151 ceph-mon[64897]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 85 B/s wr, 119 op/s 2024-04-27T02:35:49.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:49 smithi026 ceph-mon[82260]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 85 B/s wr, 119 op/s 2024-04-27T02:35:49.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:49 smithi026 ceph-mon[87214]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 85 B/s wr, 119 op/s 2024-04-27T02:35:51.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:51 smithi151 ceph-mon[64897]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 217 op/s 2024-04-27T02:35:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:51 smithi026 ceph-mon[82260]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 217 op/s 2024-04-27T02:35:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:51 smithi026 ceph-mon[87214]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 217 op/s 2024-04-27T02:35:52.353 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:35:52] "GET /metrics HTTP/1.1" 200 38304 "" "Prometheus/2.43.0" 2024-04-27T02:35:53.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[82260]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 217 op/s 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[87214]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 217 op/s 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:52 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:52 smithi151 ceph-mon[64897]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 217 op/s 2024-04-27T02:35:53.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:52 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:52 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:52 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:53.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:52 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:53 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:54.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:53 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:55.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:55 smithi026 ceph-mon[82260]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-04-27T02:35:55.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:55 smithi026 ceph-mon[87214]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-04-27T02:35:55.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:55 smithi151 ceph-mon[64897]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-04-27T02:35:56.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:56.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:56.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:35:57.745 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 136 KiB/s rd, 170 B/s wr, 215 op/s 2024-04-27T02:35:57.745 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.745 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.745 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.745 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.oulbnt"}]: dispatch 2024-04-27T02:35:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.oulbnt"}]': finished 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi151.blhads"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi151.blhads"}]': finished 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-27T02:35:57.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.749 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.751 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:57 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 136 KiB/s rd, 170 B/s wr, 215 op/s 2024-04-27T02:35:57.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.oulbnt"}]: dispatch 2024-04-27T02:35:57.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.oulbnt"}]': finished 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi151.blhads"}]: dispatch 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi151.blhads"}]': finished 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:35:57.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 136 KiB/s rd, 170 B/s wr, 215 op/s 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.857 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.oulbnt"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.oulbnt"}]': finished 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi151.blhads"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi151.blhads"}]': finished 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-27T02:35:57.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:35:57.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:57 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all rgw 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nfs 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Upgrade: Finalizing container_image settings 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Upgrade: Complete! 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: Checking dashboard <-> RGW credentials 2024-04-27T02:35:58.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:58 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:58.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all rgw 2024-04-27T02:35:58.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:35:58.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:35:58.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nfs 2024-04-27T02:35:58.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:35:58.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Upgrade: Finalizing container_image settings 2024-04-27T02:35:58.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Upgrade: Complete! 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: Checking dashboard <-> RGW credentials 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all rgw 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nfs 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all nvmeof 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Upgrade: Finalizing container_image settings 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Upgrade: Complete! 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: Checking dashboard <-> RGW credentials 2024-04-27T02:35:58.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:58 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:35:59.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:35:59 smithi151 ceph-mon[64897]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 85 B/s wr, 135 op/s 2024-04-27T02:35:59.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:35:59 smithi026 ceph-mon[82260]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 85 B/s wr, 135 op/s 2024-04-27T02:35:59.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:35:59 smithi026 ceph-mon[87214]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 85 B/s wr, 135 op/s 2024-04-27T02:36:01.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:01 smithi151 ceph-mon[64897]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 85 B/s wr, 131 op/s 2024-04-27T02:36:01.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:01 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:01.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:01 smithi026 ceph-mon[82260]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 85 B/s wr, 131 op/s 2024-04-27T02:36:01.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:01 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:01.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:01 smithi026 ceph-mon[87214]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 85 B/s wr, 131 op/s 2024-04-27T02:36:01.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:01 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:02.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:36:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:36:02] "GET /metrics HTTP/1.1" 200 38337 "" "Prometheus/2.43.0" 2024-04-27T02:36:03.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:03 smithi151 ceph-mon[64897]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:36:03.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:03 smithi026 ceph-mon[82260]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:36:03.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:03 smithi026 ceph-mon[87214]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:36:05.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:05 smithi151 ceph-mon[64897]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:36:05.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:05 smithi026 ceph-mon[82260]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:36:05.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:05 smithi026 ceph-mon[87214]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 33 op/s 2024-04-27T02:36:07.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:07 smithi151 ceph-mon[64897]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-27T02:36:07.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:07 smithi026 ceph-mon[82260]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-27T02:36:07.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:07 smithi026 ceph-mon[87214]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-27T02:36:08.587 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:08 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:08.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:08 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:08.791 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:08 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:08.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:36:09.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:09 smithi026 ceph-mon[82260]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-27T02:36:09.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:09 smithi026 ceph-mon[82260]: from='client.54682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:09.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:09 smithi026 ceph-mon[87214]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-27T02:36:09.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:09 smithi026 ceph-mon[87214]: from='client.54682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:09.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:09 smithi151 ceph-mon[64897]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-27T02:36:09.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:09 smithi151 ceph-mon[64897]: from='client.54682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:11.061 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:36:11.061 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (10m) 19s ago 32m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:36:11.061 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (9m) 19s ago 31m 77.5M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:36:11.061 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (10m) 19s ago 31m 64.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (11m) 19s ago 36m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (21m) 19s ago 38m 513M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (7m) 19s ago 38m 61.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (8m) 19s ago 36m 51.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (7m) 19s ago 36m 48.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (10m) 19s ago 32m 16.5M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (10m) 19s ago 32m 15.7M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (6m) 19s ago 36m 60.1M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (5m) 19s ago 35m 56.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (4m) 19s ago 35m 48.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 19s ago 35m 54.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (3m) 19s ago 34m 61.7M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (2m) 19s ago 34m 56.2M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:36:11.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (2m) 19s ago 33m 56.6M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:36:11.063 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (97s) 19s ago 33m 56.1M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 bec04d87416c 2024-04-27T02:36:11.063 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (10m) 19s ago 32m 71.0M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:36:11.063 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (28s) 19s ago 31m 78.1M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fca14d9b67b7 2024-04-27T02:36:11.063 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (23s) 19s ago 31m 80.3M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 23cdbd8a82c4 2024-04-27T02:36:11.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-04-27T02:36:11.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:11 smithi026 ceph-mon[82260]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-27T02:36:11.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:11.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:11 smithi026 ceph-mon[87214]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-27T02:36:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:11.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:11 smithi151 ceph-mon[64897]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-27T02:36:11.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:12.348 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:36:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:36:12] "GET /metrics HTTP/1.1" 200 38335 "" "Prometheus/2.43.0" 2024-04-27T02:36:12.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:12 smithi026 ceph-mon[82260]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:12.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:12 smithi026 ceph-mon[87214]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:12.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:12 smithi151 ceph-mon[64897]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:13.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:13 smithi026 ceph-mon[82260]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:13.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:13 smithi026 ceph-mon[87214]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:13.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:13 smithi151 ceph-mon[64897]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:13.870 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:36:14.547 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-04-27T02:36:14.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:14 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2729953019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:14.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:14 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2729953019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:14.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:14 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2729953019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:15.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:15 smithi151 ceph-mon[64897]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:15.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:15 smithi026 ceph-mon[82260]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:15.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:15 smithi026 ceph-mon[87214]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:16.734 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)" 2024-04-27T02:36:17.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:36:17.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:17 smithi026 ceph-mon[82260]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:17.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:17 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/3775107055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:17.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:17 smithi026 ceph-mon[87214]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:17.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:17 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/3775107055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:17.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:17 smithi151 ceph-mon[64897]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:17.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:17 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/3775107055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:18.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:18 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:18.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:18 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:18.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:18 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:19.614 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:36:19.615 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:36:19.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:19 smithi151 ceph-mon[64897]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:19.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:19 smithi026 ceph-mon[82260]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:19.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:19 smithi026 ceph-mon[87214]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:20.246 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:36:20.780 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:20 smithi026 ceph-mon[82260]: from='client.54706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:20.780 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:20 smithi026 ceph-mon[87214]: from='client.54706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:20.842 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:20 smithi151 ceph-mon[64897]: from='client.54706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:21.793 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:21 smithi151 ceph-mon[64897]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:21.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:21 smithi026 ceph-mon[82260]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:21.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:21 smithi026 ceph-mon[87214]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:22.093 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:36:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=cleanup t=2024-04-27T02:36:21.81939785Z level=info msg="Completed cleanup jobs" duration=3.532109ms 2024-04-27T02:36:22.093 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:36:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=sqlstore.transactions t=2024-04-27T02:36:21.827193838Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-27T02:36:22.342 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:36:22.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:36:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:36:22] "GET /metrics HTTP/1.1" 200 38335 "" "Prometheus/2.43.0" 2024-04-27T02:36:22.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:22 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/4055659223' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:36:22.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:22 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/4055659223' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:36:22.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:22 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/4055659223' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:36:22.968 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-27T02:36:23.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:23 smithi026 ceph-mon[82260]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:23.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:23 smithi026 ceph-mon[87214]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:23.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:23 smithi151 ceph-mon[64897]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:24.851 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:36:25.470 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-27T02:36:25.481 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-04-27T02:36:25.481 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[87214]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[82260]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:25.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:25.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:25.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:25.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:36:25.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:25 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:25.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:25 smithi151 ceph-mon[64897]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:25.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:25.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:25.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:25.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:36:25.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:25 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:26.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:26 smithi151 ceph-mon[64897]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:26.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:26 smithi151 ceph-mon[64897]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:36:26.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:26 smithi151 ceph-mon[64897]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:36:26.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:26.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[87214]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:26.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[87214]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:36:26.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[87214]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:36:26.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:26.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[82260]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:26.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[82260]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:36:26.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[82260]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-27T02:36:26.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:27.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:27 smithi026 ceph-mon[82260]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:27.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:27 smithi026 ceph-mon[87214]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:27.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:27 smithi151 ceph-mon[64897]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:28.127 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:36:28.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:28 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:28.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:28 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (10m) 36s ago 32m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (10m) 36s ago 32m 77.5M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (11m) 36s ago 31m 64.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (11m) 36s ago 36m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (21m) 36s ago 38m 513M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (8m) 36s ago 38m 61.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (9m) 36s ago 37m 51.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (7m) 36s ago 37m 48.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (11m) 36s ago 33m 16.5M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (10m) 36s ago 33m 15.7M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (6m) 36s ago 36m 60.1M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (6m) 36s ago 36m 56.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:36:28.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (5m) 36s ago 35m 48.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (4m) 36s ago 35m 54.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (3m) 36s ago 35m 61.7M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (3m) 36s ago 34m 56.2M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (2m) 36s ago 34m 56.6M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (114s) 36s ago 33m 56.1M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 bec04d87416c 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (10m) 36s ago 33m 71.0M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (46s) 36s ago 31m 78.1M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fca14d9b67b7 2024-04-27T02:36:28.835 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (41s) 36s ago 31m 80.3M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 23cdbd8a82c4 2024-04-27T02:36:28.842 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:28 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 8 2024-04-27T02:36:29.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:36:29.257 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:36:29.257 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:36:29.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:36:29.257 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:36:29.257 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 15 2024-04-27T02:36:29.257 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:36:29.257 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:36:29.635 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:36:29.635 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "osd", 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr", 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw" 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "15/21 daemons upgraded", 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading iscsi daemons", 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-27T02:36:29.636 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:36:29.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:29.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:29.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:29.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:36:29.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mgr 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/1936719095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:36:29.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mgr 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:29.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:29.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:29 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/1936719095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.055 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (squid) 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mgr 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.095 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:29 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/1936719095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:36:30.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mon 2024-04-27T02:36:30.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all crash 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all osd 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all mds 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all rgw 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2155356548' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:36:30.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mon 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all crash 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all osd 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all mds 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all rgw 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2155356548' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:36:30.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:30 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mon 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all crash 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all osd 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all mds 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all rgw 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all rbd-mirror 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: Upgrade: Setting container_image for all ceph-exporter 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:31.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2155356548' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:36:31.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:31.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.ofvlhd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-27T02:36:31.094 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:30 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:31.973 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[87214]: from='client.44808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:31.973 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[87214]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:31.973 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[87214]: Upgrade: Updating iscsi.foo.smithi026.ofvlhd 2024-04-27T02:36:31.973 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[87214]: Deploying daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:36:31.974 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[82260]: from='client.44808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:31.974 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[82260]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:31.974 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[82260]: Upgrade: Updating iscsi.foo.smithi026.ofvlhd 2024-04-27T02:36:31.974 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:31 smithi026 ceph-mon[82260]: Deploying daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:36:32.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:31 smithi151 ceph-mon[64897]: from='client.44808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:36:32.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:31 smithi151 ceph-mon[64897]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:32.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:31 smithi151 ceph-mon[64897]: Upgrade: Updating iscsi.foo.smithi026.ofvlhd 2024-04-27T02:36:32.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:31 smithi151 ceph-mon[64897]: Deploying daemon iscsi.foo.smithi026.ofvlhd on smithi026 2024-04-27T02:36:32.352 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:36:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:36:32] "GET /metrics HTTP/1.1" 200 38328 "" "Prometheus/2.43.0" 2024-04-27T02:36:34.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:33 smithi151 ceph-mon[64897]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:34.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:33 smithi026 ceph-mon[82260]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:34.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:33 smithi026 ceph-mon[87214]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:35.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:34 smithi151 ceph-mon[64897]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:35.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:34 smithi026 ceph-mon[82260]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:35.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:34 smithi026 ceph-mon[87214]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:37.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:36 smithi151 ceph-mon[64897]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:37.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:36 smithi026 ceph-mon[82260]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:37.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:36 smithi026 ceph-mon[87214]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:38.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:37 smithi151 ceph-mon[64897]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:38.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:37 smithi026 ceph-mon[82260]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:38.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:37 smithi026 ceph-mon[87214]: from='client.25501 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:36:39.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:38 smithi151 ceph-mon[64897]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:39.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:38 smithi026 ceph-mon[82260]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:39.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:38 smithi026 ceph-mon[87214]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:41.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:40 smithi151 ceph-mon[64897]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:41.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:40 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:41.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:40 smithi026 ceph-mon[82260]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:41.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:40 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:40 smithi026 ceph-mon[87214]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:36:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:40 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:42.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:36:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:36:42] "GET /metrics HTTP/1.1" 200 38337 "" "Prometheus/2.43.0" 2024-04-27T02:36:43.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:42 smithi026 ceph-mon[82260]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:43.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:42 smithi026 ceph-mon[87214]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:43.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:42 smithi151 ceph-mon[64897]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:44.274 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:36:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:36:43.926+0000 7fc797c4f700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. 2024-04-27T02:36:45.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:45.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. 2024-04-27T02:36:45.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:45.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:36:45.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:45.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:45.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:36:45.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:45.344 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:44 smithi151 ceph-mon[64897]: Health check failed: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[82260]: Health check failed: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:36:45.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. 2024-04-27T02:36:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-27T02:36:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:36:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:36:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:36:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:44 smithi026 ceph-mon[87214]: Health check failed: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-27T02:36:47.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:46 smithi151 ceph-mon[64897]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:36:47.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:46 smithi026 ceph-mon[82260]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:36:47.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:46 smithi026 ceph-mon[87214]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:36:49.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:48 smithi151 ceph-mon[64897]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:36:49.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:48 smithi026 ceph-mon[82260]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:36:49.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:48 smithi026 ceph-mon[87214]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-27T02:36:51.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:50 smithi151 ceph-mon[64897]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:51.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:50 smithi026 ceph-mon[82260]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:51.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:50 smithi026 ceph-mon[87214]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:52.507 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:36:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:36:52] "GET /metrics HTTP/1.1" 200 38337 "" "Prometheus/2.43.0" 2024-04-27T02:36:53.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:52 smithi151 ceph-mon[64897]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:53.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:52 smithi026 ceph-mon[82260]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:53.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:52 smithi026 ceph-mon[87214]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:55.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:54 smithi026 ceph-mon[82260]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:55.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:54 smithi026 ceph-mon[87214]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:55.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:54 smithi151 ceph-mon[64897]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 306 B/s rd, 0 op/s 2024-04-27T02:36:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:56 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2459089985' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:36:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:56 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/4116400025' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]: dispatch 2024-04-27T02:36:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:56 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]: dispatch 2024-04-27T02:36:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:56.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2459089985' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/4116400025' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2459089985' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/4116400025' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]: dispatch 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:36:56.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:55 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:36:57.243 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:57 smithi151 ceph-mon[64897]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:36:57.243 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:57 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]': finished 2024-04-27T02:36:57.243 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:57 smithi151 ceph-mon[64897]: osdmap e153: 8 total, 8 up, 8 in 2024-04-27T02:36:57.243 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:57 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2237869450' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]: dispatch 2024-04-27T02:36:57.243 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:57 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]: dispatch 2024-04-27T02:36:57.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[82260]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:36:57.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]': finished 2024-04-27T02:36:57.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[82260]: osdmap e153: 8 total, 8 up, 8 in 2024-04-27T02:36:57.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2237869450' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]: dispatch 2024-04-27T02:36:57.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]: dispatch 2024-04-27T02:36:57.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[87214]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-27T02:36:57.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3276819181"}]': finished 2024-04-27T02:36:57.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[87214]: osdmap e153: 8 total, 8 up, 8 in 2024-04-27T02:36:57.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2237869450' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]: dispatch 2024-04-27T02:36:57.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:57 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]: dispatch 2024-04-27T02:36:58.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:58 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]': finished 2024-04-27T02:36:58.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:58 smithi151 ceph-mon[64897]: osdmap e154: 8 total, 8 up, 8 in 2024-04-27T02:36:58.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:58 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2014000546' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]: dispatch 2024-04-27T02:36:58.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:58 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]: dispatch 2024-04-27T02:36:58.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]': finished 2024-04-27T02:36:58.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[87214]: osdmap e154: 8 total, 8 up, 8 in 2024-04-27T02:36:58.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2014000546' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]: dispatch 2024-04-27T02:36:58.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]: dispatch 2024-04-27T02:36:58.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1634379999"}]': finished 2024-04-27T02:36:58.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[82260]: osdmap e154: 8 total, 8 up, 8 in 2024-04-27T02:36:58.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2014000546' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]: dispatch 2024-04-27T02:36:58.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:58 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]: dispatch 2024-04-27T02:36:59.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:59 smithi151 ceph-mon[64897]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:36:59.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:59 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]': finished 2024-04-27T02:36:59.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:59 smithi151 ceph-mon[64897]: osdmap e155: 8 total, 8 up, 8 in 2024-04-27T02:36:59.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:59 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/385618405' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]: dispatch 2024-04-27T02:36:59.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:36:59 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]: dispatch 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[82260]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]': finished 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[82260]: osdmap e155: 8 total, 8 up, 8 in 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/385618405' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]: dispatch 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]: dispatch 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[87214]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1169324138"}]': finished 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[87214]: osdmap e155: 8 total, 8 up, 8 in 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/385618405' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]: dispatch 2024-04-27T02:36:59.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:36:59 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]: dispatch 2024-04-27T02:37:00.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:00 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]': finished 2024-04-27T02:37:00.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:00 smithi151 ceph-mon[64897]: osdmap e156: 8 total, 8 up, 8 in 2024-04-27T02:37:00.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:00 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/170773408' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]: dispatch 2024-04-27T02:37:00.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:00 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]: dispatch 2024-04-27T02:37:00.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]': finished 2024-04-27T02:37:00.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[82260]: osdmap e156: 8 total, 8 up, 8 in 2024-04-27T02:37:00.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/170773408' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]: dispatch 2024-04-27T02:37:00.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]: dispatch 2024-04-27T02:37:00.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2221496436"}]': finished 2024-04-27T02:37:00.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[87214]: osdmap e156: 8 total, 8 up, 8 in 2024-04-27T02:37:00.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/170773408' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]: dispatch 2024-04-27T02:37:00.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:00 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]: dispatch 2024-04-27T02:37:00.426 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:37:00.769 INFO:teuthology.orchestra.run.smithi026.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed." 2024-04-27T02:37:01.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[82260]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[82260]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]': finished 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[82260]: osdmap e157: 8 total, 8 up, 8 in 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2881183668' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1634379999"}]: dispatch 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[82260]: from='client.54820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[87214]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[87214]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]': finished 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[87214]: osdmap e157: 8 total, 8 up, 8 in 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2881183668' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1634379999"}]: dispatch 2024-04-27T02:37:01.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:01 smithi026 ceph-mon[87214]: from='client.54820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:01.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:01 smithi151 ceph-mon[64897]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:01.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:01 smithi151 ceph-mon[64897]: from='client.? ' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/898056195"}]': finished 2024-04-27T02:37:01.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:01 smithi151 ceph-mon[64897]: osdmap e157: 8 total, 8 up, 8 in 2024-04-27T02:37:01.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:01 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2881183668' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1634379999"}]: dispatch 2024-04-27T02:37:01.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:01 smithi151 ceph-mon[64897]: from='client.54820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:01.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:37:02.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:02 smithi026 ceph-mon[87214]: from='client.44886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:02.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:02 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2881183668' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1634379999"}]': finished 2024-04-27T02:37:02.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:02 smithi026 ceph-mon[87214]: osdmap e158: 8 total, 8 up, 8 in 2024-04-27T02:37:02.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:02 smithi026 ceph-mon[82260]: from='client.44886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:02.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:02 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2881183668' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1634379999"}]': finished 2024-04-27T02:37:02.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:02 smithi026 ceph-mon[82260]: osdmap e158: 8 total, 8 up, 8 in 2024-04-27T02:37:02.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:02 smithi151 ceph-mon[64897]: from='client.44886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:02.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:02 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2881183668' entity='client.iscsi.foo.smithi026.ofvlhd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1634379999"}]': finished 2024-04-27T02:37:02.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:02 smithi151 ceph-mon[64897]: osdmap e158: 8 total, 8 up, 8 in 2024-04-27T02:37:02.351 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:37:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:37:02] "GET /metrics HTTP/1.1" 200 38410 "" "Prometheus/2.43.0" 2024-04-27T02:37:03.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:03 smithi026 ceph-mon[82260]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:03.151 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:03 smithi026 ceph-mon[87214]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:03.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:03 smithi151 ceph-mon[64897]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (10m) 71s ago 32m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (10m) 71s ago 32m 77.5M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (11m) 71s ago 32m 64.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (11m) 71s ago 37m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (22m) 71s ago 39m 513M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (8m) 71s ago 39m 61.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (9m) 71s ago 37m 51.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (8m) 71s ago 37m 48.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (11m) 71s ago 33m 16.5M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:37:03.593 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (11m) 71s ago 33m 15.7M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (7m) 71s ago 37m 60.1M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (6m) 71s ago 36m 56.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (5m) 71s ago 36m 48.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (4m) 71s ago 35m 54.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (4m) 71s ago 35m 61.7M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (3m) 71s ago 35m 56.2M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (3m) 71s ago 34m 56.6M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (2m) 71s ago 34m 56.1M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 bec04d87416c 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (11m) 71s ago 33m 71.0M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (81s) 71s ago 32m 78.1M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fca14d9b67b7 2024-04-27T02:37:03.594 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (75s) 71s ago 32m 80.3M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 23cdbd8a82c4 2024-04-27T02:37:04.295 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:04 smithi026 ceph-mon[82260]: from='client.54832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:04.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:04 smithi026 ceph-mon[87214]: from='client.54832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:04.321 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-27T02:37:04.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:04 smithi151 ceph-mon[64897]: from='client.54832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:37:05.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:05 smithi026 ceph-mon[82260]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1 op/s 2024-04-27T02:37:05.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:05 smithi026 ceph-mon[87214]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1 op/s 2024-04-27T02:37:05.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:05 smithi151 ceph-mon[64897]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1 op/s 2024-04-27T02:37:06.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:06 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:06.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:06 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:06.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:06 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:06.358 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:37:06.358 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:37:06.358 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:37:06.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 8 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 15 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:37:06.359 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:37:07.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-27T02:37:07.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:07 smithi151 ceph-mon[64897]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:37:07.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:07 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/2217635802' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:37:07.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:07 smithi026 ceph-mon[82260]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:37:07.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:07 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/2217635802' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:37:07.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:07 smithi026 ceph-mon[87214]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-27T02:37:07.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:07 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/2217635802' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:37:08.875 INFO:teuthology.orchestra.run.smithi026.stdout:wait for servicemap items w/ changing names to refresh 2024-04-27T02:37:09.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:09 smithi151 ceph-mon[64897]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:37:09.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:09 smithi026 ceph-mon[82260]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:37:09.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:09 smithi026 ceph-mon[87214]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-27T02:37:09.626 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-27T02:37:11.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:11 smithi026 ceph-mon[87214]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:37:11.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:11.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:11 smithi026 ceph-mon[82260]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:37:11.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:11.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:11 smithi151 ceph-mon[64897]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-27T02:37:11.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:12.601 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:37:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:37:12] "GET /metrics HTTP/1.1" 200 38414 "" "Prometheus/2.43.0" 2024-04-27T02:37:13.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:13 smithi151 ceph-mon[64897]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-27T02:37:13.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:13 smithi026 ceph-mon[82260]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-27T02:37:13.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:13 smithi026 ceph-mon[87214]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-27T02:37:15.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:15 smithi151 ceph-mon[64897]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:15.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:15 smithi026 ceph-mon[82260]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:15.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:15 smithi026 ceph-mon[87214]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:16.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:16 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:16.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:16 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:16.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:16 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:17.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:17 smithi151 ceph-mon[64897]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:17.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:17 smithi026 ceph-mon[82260]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:17.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:17 smithi026 ceph-mon[87214]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:19.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:19 smithi151 ceph-mon[64897]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:37:19.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:19 smithi026 ceph-mon[82260]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:37:19.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:19 smithi026 ceph-mon[87214]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:37:21.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:21 smithi151 ceph-mon[64897]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:21.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:21 smithi026 ceph-mon[82260]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:21.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:21 smithi026 ceph-mon[87214]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:22.513 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:37:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:37:22] "GET /metrics HTTP/1.1" 200 38414 "" "Prometheus/2.43.0" 2024-04-27T02:37:23.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:23 smithi151 ceph-mon[64897]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:37:23.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:23 smithi026 ceph-mon[82260]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:37:23.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:23 smithi026 ceph-mon[87214]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:37:25.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:25 smithi151 ceph-mon[64897]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:25.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:25 smithi026 ceph-mon[82260]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:25.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:25 smithi026 ceph-mon[87214]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:26.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:26 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:26.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:26.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:26 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:26.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:26.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:26 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:26.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:27.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:27 smithi151 ceph-mon[64897]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:27.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:27 smithi026 ceph-mon[82260]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:27.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:27 smithi026 ceph-mon[87214]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:29.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:29 smithi151 ceph-mon[64897]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:29.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:29 smithi026 ceph-mon[82260]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:29.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:29 smithi026 ceph-mon[87214]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:31.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:31 smithi151 ceph-mon[64897]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:31.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:31 smithi026 ceph-mon[82260]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:31.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:31 smithi026 ceph-mon[87214]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:32.601 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:37:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:37:32] "GET /metrics HTTP/1.1" 200 38417 "" "Prometheus/2.43.0" 2024-04-27T02:37:33.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:33 smithi151 ceph-mon[64897]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:33.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:33 smithi026 ceph-mon[82260]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:33.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:33 smithi026 ceph-mon[87214]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:35.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:35 smithi151 ceph-mon[64897]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:35.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:35 smithi026 ceph-mon[82260]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:35.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:35 smithi026 ceph-mon[87214]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:36.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:36 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:36.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:36 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:36.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:36 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:37.342 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:37 smithi151 ceph-mon[64897]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:37.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:37 smithi026 ceph-mon[82260]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:37.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:37 smithi026 ceph-mon[87214]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:39.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:39 smithi151 ceph-mon[64897]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:39.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:39 smithi026 ceph-mon[82260]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:39.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:39 smithi026 ceph-mon[87214]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:41.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:41 smithi151 ceph-mon[64897]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:41.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:41 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:41.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:41 smithi026 ceph-mon[82260]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:41.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:41 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:41.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:41 smithi026 ceph-mon[87214]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:41 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:42.602 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:37:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:37:42] "GET /metrics HTTP/1.1" 200 38415 "" "Prometheus/2.43.0" 2024-04-27T02:37:43.343 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:43 smithi151 ceph-mon[64897]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:43.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:43 smithi026 ceph-mon[82260]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:43.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:43 smithi026 ceph-mon[87214]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:44.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:44 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:37:44.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:44 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:37:44.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:44 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-27T02:37:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[82260]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:37:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:37:45.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:37:45.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[87214]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:45.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:37:45.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:37:45.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:45 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:37:45.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:45 smithi151 ceph-mon[64897]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:45.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-27T02:37:45.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-27T02:37:45.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:45 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' 2024-04-27T02:37:46.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:46 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:46.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:46 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:46.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:46 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:47.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:47 smithi026 ceph-mon[82260]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:47.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:47 smithi026 ceph-mon[87214]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:47.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:47 smithi151 ceph-mon[64897]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:49.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:49 smithi151 ceph-mon[64897]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:49.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:49 smithi026 ceph-mon[82260]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:49.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:49 smithi026 ceph-mon[87214]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:51.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:51 smithi151 ceph-mon[64897]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:51.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:51 smithi026 ceph-mon[82260]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:51 smithi026 ceph-mon[87214]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:52.503 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:37:52 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:37:52] "GET /metrics HTTP/1.1" 200 38415 "" "Prometheus/2.43.0" 2024-04-27T02:37:53.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:53 smithi026 ceph-mon[82260]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:53.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:53 smithi026 ceph-mon[87214]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:53.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:53 smithi151 ceph-mon[64897]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:55.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:55 smithi151 ceph-mon[64897]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:55.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:55 smithi026 ceph-mon[82260]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:55.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:55 smithi026 ceph-mon[87214]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:56.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:56 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:56.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:56 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:56.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:56 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:56.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:56 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:56.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:56 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:37:56.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:56 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:37:57.494 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:57 smithi151 ceph-mon[64897]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:57.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:57 smithi026 ceph-mon[82260]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:57.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:57 smithi026 ceph-mon[87214]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:37:59.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:37:59 smithi151 ceph-mon[64897]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:59.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:37:59 smithi026 ceph-mon[82260]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:37:59.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:37:59 smithi026 ceph-mon[87214]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:01.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:01 smithi151 ceph-mon[64897]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:01.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:01 smithi026 ceph-mon[82260]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:01.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:01 smithi026 ceph-mon[87214]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:02.601 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:38:02] "GET /metrics HTTP/1.1" 200 38415 "" "Prometheus/2.43.0" 2024-04-27T02:38:03.594 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:03 smithi151 ceph-mon[64897]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:03.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:03 smithi026 ceph-mon[82260]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:03.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:03 smithi026 ceph-mon[87214]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:05.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:05 smithi151 ceph-mon[64897]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:05.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:05 smithi026 ceph-mon[82260]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:05.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:05 smithi026 ceph-mon[87214]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:06.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:06 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:06.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:06 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:06.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:06 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:07.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:07 smithi151 ceph-mon[64897]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:07.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:07 smithi026 ceph-mon[82260]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:07.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:07 smithi026 ceph-mon[87214]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:09.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:09 smithi151 ceph-mon[64897]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:09.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:09 smithi026 ceph-mon[82260]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:09.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:09 smithi026 ceph-mon[87214]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:11.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:11 smithi151 ceph-mon[64897]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:11.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:11 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:38:11.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:11 smithi026 ceph-mon[82260]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:11.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:11 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:38:11.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:11 smithi026 ceph-mon[87214]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:11.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:11 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:38:12.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-27T02:38:12.576 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:12 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:38:12] "GET /metrics HTTP/1.1" 200 38419 "" "Prometheus/2.43.0" 2024-04-27T02:38:13.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:13 smithi026 ceph-mon[82260]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:38:13.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:13 smithi026 ceph-mon[87214]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:38:13.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:13 smithi151 ceph-mon[64897]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (12m) 2m ago 34m 25.5M - 0.25.0 c8568f914cd2 8b1e48c88073 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi151 *:3000 running (11m) 2m ago 33m 77.5M - 9.4.12 2bacad6d85d8 89665f31c148 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.ofvlhd smithi026 running (12m) 2m ago 33m 64.5M - 3.5 e1d6a67b021e ca02a6e03921 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi151 *:8443,9283,8765 running (13m) 2m ago 38m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 06ab5f381980 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (23m) 2m ago 40m 513M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b4373d6f8791 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (9m) 2m ago 40m 61.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 576ac55bff7b 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi151 running (10m) 2m ago 38m 51.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 cb8177cb8404 2024-04-27T02:38:14.359 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (9m) 2m ago 39m 48.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0757865e9908 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (12m) 2m ago 34m 16.5M - 1.5.0 0da6a335fe13 a01cf397c18a 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi151 *:9100 running (12m) 2m ago 34m 15.7M - 1.5.0 0da6a335fe13 881b4b216b6b 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) 2m ago 38m 60.1M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 860b174a16df 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (7m) 2m ago 37m 56.8M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 00c20346f672 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (6m) 2m ago 37m 48.7M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 77827e7c5612 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (5m) 2m ago 37m 54.2M 2993M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a9b308864afa 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi151 running (5m) 2m ago 36m 61.7M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5f13d235f4f4 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi151 running (4m) 2m ago 36m 56.2M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d15baed129e 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi151 running (4m) 2m ago 36m 56.6M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 88ba76966daf 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi151 running (3m) 2m ago 35m 56.1M 3248M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 bec04d87416c 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi151 *:9095 running (12m) 2m ago 34m 71.0M - 2.43.0 a07b618ecd1d 53ac67210d3b 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.oulbnt smithi026 *:8000 running (2m) 2m ago 33m 78.1M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fca14d9b67b7 2024-04-27T02:38:14.360 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi151.blhads smithi151 *:8000 running (2m) 2m ago 33m 80.3M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 23cdbd8a82c4 2024-04-27T02:38:15.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-27T02:38:15.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:15 smithi026 ceph-mon[82260]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:15.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:15 smithi026 ceph-mon[82260]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:15.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:15 smithi026 ceph-mon[87214]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:15.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:15 smithi026 ceph-mon[87214]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:15.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:15 smithi151 ceph-mon[64897]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:15.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:15 smithi151 ceph-mon[64897]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:16.374 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:16 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:16.374 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:16 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:16.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:16 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:17.215 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:38:17.215 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 8 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 15 2024-04-27T02:38:17.216 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-27T02:38:17.217 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:38:17.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:17 smithi026 ceph-mon[82260]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:17.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:17 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/1789370565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:17.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:17 smithi026 ceph-mon[87214]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:17.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:17 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/1789370565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:17.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:17 smithi151 ceph-mon[64897]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:17.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:17 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/1789370565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:17.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade status' 2024-04-27T02:38:19.394 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:19 smithi026 ceph-mon[82260]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:19.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:19 smithi026 ceph-mon[87214]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:19.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:19 smithi151 ceph-mon[64897]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:19.911 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6", 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "osd", 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr", 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw" 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "15/21 daemons upgraded", 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed.", 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": true 2024-04-27T02:38:19.912 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:38:20.550 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph health detail' 2024-04-27T02:38:21.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:21 smithi151 ceph-mon[64897]: from='client.54856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:21.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:21 smithi151 ceph-mon[64897]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:21.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:21 smithi026 ceph-mon[82260]: from='client.54856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:21.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:21 smithi026 ceph-mon[82260]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:21.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:21 smithi026 ceph-mon[87214]: from='client.54856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:21.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:21 smithi026 ceph-mon[87214]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:22.601 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:22 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:38:22] "GET /metrics HTTP/1.1" 200 38419 "" "Prometheus/2.43.0" 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.ofvlhd on host smithi026 failed. 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrade daemon: iscsi.foo.smithi026.ofvlhd: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi026.ofvlhd ... 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout:Creating ceph-iscsi config... 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout:Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/iscsi-gateway.cfg 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout:Write file: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/iscsi.foo.smithi026.ofvlhd/tcmu-runner-entrypoint.sh 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d7a5462be\x2d0439\x2d11ef\x2dbc93\x2dc7b262605968.slice/ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service 2024-04-27T02:38:22.630 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl start ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout:Traceback (most recent call last): 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: exec(code, run_globals) 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5608, in 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 5596, in main 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3096, in command_deploy_from 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3131, in _common_deploy 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1087, in deploy_daemon 2024-04-27T02:38:22.631 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/__main__.py", line 1223, in deploy_daemon_units 2024-04-27T02:38:22.632 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/cephadm.ba9cadcb2fa5feb5b2b12c83b0bf8306a2b6111ad6873a4e96a2a3fe4f34629b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-27T02:38:22.632 INFO:teuthology.orchestra.run.smithi026.stdout:RuntimeError: Failed command: systemctl start ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd: Job for ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service failed because the control process exited with error code. 2024-04-27T02:38:22.632 INFO:teuthology.orchestra.run.smithi026.stdout:See "systemctl status ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" and "journalctl -xeu ceph-7a5462be-0439-11ef-bc93-c7b262605968@iscsi.foo.smithi026.ofvlhd.service" for details. 2024-04-27T02:38:23.214 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-27T02:38:23.530 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:23 smithi026 ceph-mon[87214]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:23.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:23 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/3890365970' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:38:23.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:23 smithi026 ceph-mon[82260]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:23.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:23 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/3890365970' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:38:23.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:23 smithi151 ceph-mon[64897]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:23.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:23 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/3890365970' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-27T02:38:25.213 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-27T02:38:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:25 smithi151 ceph-mon[64897]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:25 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/562548197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:25.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:25 smithi026 ceph-mon[82260]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:25.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:25 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/562548197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:25.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:25 smithi026 ceph-mon[87214]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:25.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:25 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/562548197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:25.947 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-27T02:38:26.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:26 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:26.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:26 smithi026 ceph-mon[87214]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:38:26.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:26 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:26.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:26 smithi026 ceph-mon[82260]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:38:26.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:26 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:26.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:26 smithi151 ceph-mon[64897]: from='mgr.25738 172.21.15.26:0/1738204818' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-27T02:38:27.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:27 smithi026 ceph-mon[82260]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:27.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:27 smithi026 ceph-mon[87214]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:27.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:27 smithi151 ceph-mon[64897]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:28.000 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)" 2024-04-27T02:38:28.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:28 smithi026 ceph-mon[82260]: from='client.? 172.21.15.26:0/4170292033' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:28.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:28 smithi026 ceph-mon[87214]: from='client.? 172.21.15.26:0/4170292033' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:28.579 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-04-27T02:38:28.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:28 smithi151 ceph-mon[64897]: from='client.? 172.21.15.26:0/4170292033' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-27T02:38:29.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:29 smithi151 ceph-mon[64897]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:29.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:29 smithi026 ceph-mon[82260]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:29.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:29 smithi026 ceph-mon[87214]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:30.483 INFO:teuthology.orchestra.run.smithi026.stdout:osd 8 2m ago - 2024-04-27T02:38:31.087 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-27T02:38:31.097 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-04-27T02:38:31.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-04-27T02:38:31.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:31 smithi026 ceph-mon[82260]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:31.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:31 smithi026 ceph-mon[82260]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:31.523 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:31 smithi026 ceph-mon[87214]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:31.523 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:31 smithi026 ceph-mon[87214]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:31.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:31 smithi151 ceph-mon[64897]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:31.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:31 smithi151 ceph-mon[64897]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:32.242 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:32 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:38:32] "GET /metrics HTTP/1.1" 200 38414 "" "Prometheus/2.43.0" 2024-04-27T02:38:33.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:33 smithi151 ceph-mon[64897]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:33.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:33 smithi026 ceph-mon[82260]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:33.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:33 smithi026 ceph-mon[87214]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:33.604 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-27T02:38:33.604 INFO:teuthology.orchestra.run.smithi026.stdout: "image": "quay.io/ceph/ceph", 2024-04-27T02:38:33.604 INFO:teuthology.orchestra.run.smithi026.stdout: "registry": "quay.io", 2024-04-27T02:38:33.604 INFO:teuthology.orchestra.run.smithi026.stdout: "bare_image": "ceph/ceph", 2024-04-27T02:38:33.605 INFO:teuthology.orchestra.run.smithi026.stdout: "versions": [] 2024-04-27T02:38:33.605 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-27T02:38:34.193 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-04-27T02:38:34.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:34 smithi026 ceph-mon[82260]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:34.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:34 smithi026 ceph-mon[87214]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:34.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:34 smithi151 ceph-mon[64897]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:35.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:35 smithi026 ceph-mon[82260]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:35.530 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:35 smithi026 ceph-mon[87214]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:35.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:35 smithi151 ceph-mon[64897]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:36.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:36 smithi151 ceph-mon[64897]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:36.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:36 smithi026 ceph-mon[87214]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:36.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:36 smithi026 ceph-mon[82260]: from='client.54769 -' entity='client.iscsi.foo.smithi026.ofvlhd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-27T02:38:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "16.2.0", 2024-04-27T02:38:37.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:37 smithi026 ceph-mon[82260]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:37.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:37 smithi026 ceph-mon[82260]: from='client.44952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:37.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:37 smithi026 ceph-mon[87214]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:37.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:37 smithi026 ceph-mon[87214]: from='client.44952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:37.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7a5462be-0439-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-04-27T02:38:37.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:37 smithi151 ceph-mon[64897]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-27T02:38:37.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:37 smithi151 ceph-mon[64897]: from='client.44952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:39.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:39 smithi151 ceph-mon[64897]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:39.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:39 smithi026 ceph-mon[82260]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:39.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:39 smithi026 ceph-mon[87214]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-27T02:38:39.892 INFO:teuthology.orchestra.run.smithi026.stdout: "v16.2.2", 2024-04-27T02:38:39.893 INFO:teuthology.orchestra.run.smithi026.stdout: "v16.2.2-20210505", 2024-04-27T02:38:40.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-mon[82260]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:40.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-mon[87214]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:40.456 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-27T02:38:40.466 INFO:tasks.cephadm:Teardown begin 2024-04-27T02:38:40.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-27T02:38:40.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-27T02:38:40.527 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-27T02:38:40.527 DEBUG:teuthology.orchestra.run.smithi026:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-27T02:38:40.551 DEBUG:teuthology.orchestra.run.smithi151:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-27T02:38:40.584 INFO:tasks.cephadm:Stopping all daemons... 2024-04-27T02:38:40.585 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-27T02:38:40.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a 2024-04-27T02:38:40.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:40 smithi151 ceph-mon[64897]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-27T02:38:40.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:40 smithi026 systemd[1]: Stopping Ceph mon.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:38:41.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a[82256]: 2024-04-27T02:38:40.936+0000 7fdf150ed700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:38:41.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a[82256]: 2024-04-27T02:38:40.936+0000 7fdf150ed700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-27T02:38:41.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:40 smithi026 podman[129088]: 2024-04-27 02:38:40.987868013 +0000 UTC m=+0.177115007 container died 576ac55bff7b36784cc0bbc28fa2f198d97cf2a7fc93b1128b64c15f82f2151c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-27T02:38:41.247 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:38:40] ENGINE Bus STOPPING 2024-04-27T02:38:41.247 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:38:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-27T02:38:41.247 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:38:40] ENGINE Bus STOPPED 2024-04-27T02:38:41.248 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:40 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:38:40] ENGINE Bus STARTING 2024-04-27T02:38:41.248 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: CherryPy Checker: 2024-04-27T02:38:41.248 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: The Application mounted at '' has an empty config. 2024-04-27T02:38:41.248 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:38:41.248 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:38:41] ENGINE Serving on http://:::9283 2024-04-27T02:38:41.248 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:41 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: [27/Apr/2024:02:38:41] ENGINE Bus STARTED 2024-04-27T02:38:41.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:41 smithi026 podman[129088]: 2024-04-27 02:38:41.373565742 +0000 UTC m=+0.562812736 container cleanup 576ac55bff7b36784cc0bbc28fa2f198d97cf2a7fc93b1128b64c15f82f2151c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:38:41.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 27 02:38:41 smithi026 bash[129088]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-a 2024-04-27T02:38:41.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.a.service' 2024-04-27T02:38:41.772 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:38:41.773 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-27T02:38:41.773 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-27T02:38:41.773 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.c 2024-04-27T02:38:41.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:41 smithi026 systemd[1]: Stopping Ceph mon.c for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:38:42.510 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: ::ffff:172.21.15.151 - - [27/Apr/2024:02:38:42] "GET /metrics HTTP/1.1" 200 38418 "" "Prometheus/2.43.0" 2024-04-27T02:38:42.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c[87210]: 2024-04-27T02:38:42.210+0000 7f9a2a580700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:38:42.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:42 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c[87210]: 2024-04-27T02:38:42.210+0000 7f9a2a580700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-04-27T02:38:42.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:42 smithi026 podman[129221]: 2024-04-27 02:38:42.262696896 +0000 UTC m=+0.280544475 container died 0757865e99087bf49c382685996c8705e2d4384080bfe5fc75b834781aaa247b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-27T02:38:42.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:42 smithi026 podman[129221]: 2024-04-27 02:38:42.653120752 +0000 UTC m=+0.670968329 container cleanup 0757865e99087bf49c382685996c8705e2d4384080bfe5fc75b834781aaa247b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-27T02:38:42.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 27 02:38:42 smithi026 bash[129221]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-c 2024-04-27T02:38:42.977 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.c.service' 2024-04-27T02:38:43.011 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:38:43.011 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-27T02:38:43.012 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-27T02:38:43.012 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.b 2024-04-27T02:38:43.319 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:43 smithi151 systemd[1]: Stopping Ceph mon.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:38:43.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b[64888]: 2024-04-27T02:38:43.345+0000 7f7c78955700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:38:43.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b[64888]: 2024-04-27T02:38:43.345+0000 7f7c78955700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-04-27T02:38:43.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:43 smithi151 podman[98932]: 2024-04-27 02:38:43.458569766 +0000 UTC m=+0.218594766 container died cb8177cb84042c8fac73fc0fa7ed1b7e779f783f79ee475fc4059441df9511ca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-27T02:38:44.092 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:43 smithi151 podman[98932]: 2024-04-27 02:38:43.921914812 +0000 UTC m=+0.681939826 container cleanup cb8177cb84042c8fac73fc0fa7ed1b7e779f783f79ee475fc4059441df9511ca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:38:44.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 27 02:38:43 smithi151 bash[98932]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mon-b 2024-04-27T02:38:44.354 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mon.b.service' 2024-04-27T02:38:44.397 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:38:44.397 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-27T02:38:44.397 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-27T02:38:44.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y 2024-04-27T02:38:44.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:44 smithi026 systemd[1]: Stopping Ceph mgr.y for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:38:45.015 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:38:44.723+0000 7fc805d89700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:38:45.015 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:44 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y[60486]: 2024-04-27T02:38:44.723+0000 7fc805d89700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-27T02:38:45.015 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:44 smithi026 podman[129343]: 2024-04-27 02:38:44.755607417 +0000 UTC m=+0.132991682 container died b4373d6f87916ff93762fcf262c4106e6c03f2082c1a66ce90321c11a0c70e5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-27T02:38:45.348 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:45 smithi026 podman[129343]: 2024-04-27 02:38:45.149537817 +0000 UTC m=+0.526922083 container cleanup b4373d6f87916ff93762fcf262c4106e6c03f2082c1a66ce90321c11a0c70e5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True) 2024-04-27T02:38:45.348 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 27 02:38:45 smithi026 bash[129343]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-y 2024-04-27T02:38:45.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-27T02:38:45.511 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:38:45.512 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-27T02:38:45.512 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-27T02:38:45.512 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x 2024-04-27T02:38:45.839 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:38:45 smithi151 systemd[1]: Stopping Ceph mgr.x for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:38:46.092 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:38:45 smithi151 podman[99053]: 2024-04-27 02:38:45.866564182 +0000 UTC m=+0.133877405 container died 06ab5f381980ffd80bf51ba80158b6b341667f5bb539ed75e737c8a288acc908 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-27T02:38:46.494 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:38:46 smithi151 podman[99053]: 2024-04-27 02:38:46.253046296 +0000 UTC m=+0.520359524 container cleanup 06ab5f381980ffd80bf51ba80158b6b341667f5bb539ed75e737c8a288acc908 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5) 2024-04-27T02:38:46.494 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 27 02:38:46 smithi151 bash[99053]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-mgr-x 2024-04-27T02:38:46.655 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-27T02:38:46.688 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:38:46.689 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-27T02:38:46.689 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-27T02:38:46.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.0 2024-04-27T02:38:46.985 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:46 smithi026 systemd[1]: Stopping Ceph osd.0 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:38:47.351 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[94629]: 2024-04-27T02:38:47.011+0000 7f7a176cf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:38:47.352 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[94629]: 2024-04-27T02:38:47.011+0000 7f7a176cf700 -1 osd.0 158 *** Got signal Terminated *** 2024-04-27T02:38:47.352 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:47 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0[94629]: 2024-04-27T02:38:47.011+0000 7f7a176cf700 -1 osd.0 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:38:50.593 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:38:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:38:50.231Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:38:50.593 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:38:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:38:50.231Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:38:50.593 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:38:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:38:50.231Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:38:50.593 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:38:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:38:50.232Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:38:50.593 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:38:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:38:50.232Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:38:52.711 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:52 smithi026 podman[129463]: 2024-04-27 02:38:52.480446534 +0000 UTC m=+5.571796674 container died 860b174a16df4bb6eba3e8c018be6a448dd8a57ba7d68f7be72b38bcd6348771 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-27T02:38:53.096 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:52 smithi026 podman[129463]: 2024-04-27 02:38:52.836837306 +0000 UTC m=+5.928187449 container cleanup 860b174a16df4bb6eba3e8c018be6a448dd8a57ba7d68f7be72b38bcd6348771 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-27T02:38:53.096 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:52 smithi026 bash[129463]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0 2024-04-27T02:38:53.351 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:53 smithi026 podman[129484]: 2024-04-27 02:38:53.123608206 +0000 UTC m=+0.639689871 container remove 860b174a16df4bb6eba3e8c018be6a448dd8a57ba7d68f7be72b38bcd6348771 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-27T02:38:53.733 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:38:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:38:53.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:38:53.733 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:38:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:38:53.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:38:53.733 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:53 smithi026 podman[129576]: 2024-04-27 02:38:53.359915996 +0000 UTC m=+0.018455711 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:38:53.733 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:53 smithi026 podman[129576]: 2024-04-27 02:38:53.483011586 +0000 UTC m=+0.141563378 container create 0b6b08509013991638aecf239391286534e29b60ce1783237a87287f79c7de1d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-27T02:38:53.733 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:53 smithi026 podman[129576]: 2024-04-27 02:38:53.70201896 +0000 UTC m=+0.360558673 container init 0b6b08509013991638aecf239391286534e29b60ce1783237a87287f79c7de1d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:38:53.733 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:53 smithi026 podman[129576]: 2024-04-27 02:38:53.70514689 +0000 UTC m=+0.363686603 container start 0b6b08509013991638aecf239391286534e29b60ce1783237a87287f79c7de1d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-27T02:38:53.733 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:53 smithi026 podman[129576]: 2024-04-27 02:38:53.760241273 +0000 UTC m=+0.418781002 container attach 0b6b08509013991638aecf239391286534e29b60ce1783237a87287f79c7de1d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-27T02:38:54.007 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 27 02:38:54 smithi026 podman[129576]: 2024-04-27 02:38:54.009812013 +0000 UTC m=+0.668351725 container died 0b6b08509013991638aecf239391286534e29b60ce1783237a87287f79c7de1d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:38:54.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.0.service' 2024-04-27T02:38:54.607 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:38:54.607 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-27T02:38:54.607 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-27T02:38:54.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.1 2024-04-27T02:38:54.851 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:38:54 smithi026 systemd[1]: Stopping Ceph osd.1 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:38:55.221 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:38:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[99787]: 2024-04-27T02:38:54.968+0000 7f6071bb7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:38:55.222 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:38:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[99787]: 2024-04-27T02:38:54.968+0000 7f6071bb7700 -1 osd.1 158 *** Got signal Terminated *** 2024-04-27T02:38:55.222 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:38:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1[99787]: 2024-04-27T02:38:54.968+0000 7f6071bb7700 -1 osd.1 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:39:00.768 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:00 smithi026 podman[129693]: 2024-04-27 02:39:00.5134918 +0000 UTC m=+5.645493270 container died 00c20346f672b996481ad11d9ca15d59beda99f964d9a3e7d25883342e4cd9f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:39:01.074 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:00 smithi026 podman[129693]: 2024-04-27 02:39:00.88400322 +0000 UTC m=+6.016004696 container cleanup 00c20346f672b996481ad11d9ca15d59beda99f964d9a3e7d25883342e4cd9f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-27T02:39:01.074 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:00 smithi026 bash[129693]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1 2024-04-27T02:39:01.351 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:01 smithi026 podman[129714]: 2024-04-27 02:39:01.101674708 +0000 UTC m=+0.583308760 container remove 00c20346f672b996481ad11d9ca15d59beda99f964d9a3e7d25883342e4cd9f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-27T02:39:01.670 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:01 smithi026 podman[129805]: 2024-04-27 02:39:01.305505906 +0000 UTC m=+0.018666850 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:39:01.670 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:01 smithi026 podman[129805]: 2024-04-27 02:39:01.42776175 +0000 UTC m=+0.140922716 container create 9048848b42fc1db95af65a8682e3f0f8737d224fa94c49ae2d09fa44d4c10ce0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:39:01.670 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:01 smithi026 podman[129805]: 2024-04-27 02:39:01.613478035 +0000 UTC m=+0.326638959 container init 9048848b42fc1db95af65a8682e3f0f8737d224fa94c49ae2d09fa44d4c10ce0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb) 2024-04-27T02:39:01.670 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:01 smithi026 podman[129805]: 2024-04-27 02:39:01.616745672 +0000 UTC m=+0.329906599 container start 9048848b42fc1db95af65a8682e3f0f8737d224fa94c49ae2d09fa44d4c10ce0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5) 2024-04-27T02:39:01.670 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:01 smithi026 podman[129805]: 2024-04-27 02:39:01.696967265 +0000 UTC m=+0.410128201 container attach 9048848b42fc1db95af65a8682e3f0f8737d224fa94c49ae2d09fa44d4c10ce0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-27T02:39:02.101 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 27 02:39:01 smithi026 podman[129805]: 2024-04-27 02:39:01.930217864 +0000 UTC m=+0.643378793 container died 9048848b42fc1db95af65a8682e3f0f8737d224fa94c49ae2d09fa44d4c10ce0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:39:02.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.1.service' 2024-04-27T02:39:02.568 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:02.568 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-27T02:39:02.569 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-27T02:39:02.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.2 2024-04-27T02:39:02.851 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:02 smithi026 systemd[1]: Stopping Ceph osd.2 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:03.351 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[106495]: 2024-04-27T02:39:02.898+0000 7fd61ae9e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:39:03.351 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[106495]: 2024-04-27T02:39:02.898+0000 7fd61ae9e700 -1 osd.2 158 *** Got signal Terminated *** 2024-04-27T02:39:03.351 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:02 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2[106495]: 2024-04-27T02:39:02.898+0000 7fd61ae9e700 -1 osd.2 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:39:03.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:03.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:03.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:03.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:03.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:03 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:03.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:08.686 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:08 smithi026 podman[129911]: 2024-04-27 02:39:08.441006403 +0000 UTC m=+5.620862416 container died 77827e7c5612a78a293678c4d574b7ef7838f577e139d9c6dbfc17f7f6f5f809 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-27T02:39:09.067 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:08 smithi026 podman[129911]: 2024-04-27 02:39:08.810638651 +0000 UTC m=+5.990494656 container cleanup 77827e7c5612a78a293678c4d574b7ef7838f577e139d9c6dbfc17f7f6f5f809 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux ) 2024-04-27T02:39:09.068 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:08 smithi026 bash[129911]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2 2024-04-27T02:39:09.351 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:09 smithi026 podman[129929]: 2024-04-27 02:39:09.095284023 +0000 UTC m=+0.652533787 container remove 77827e7c5612a78a293678c4d574b7ef7838f577e139d9c6dbfc17f7f6f5f809 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:39:09.738 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:09 smithi026 podman[130024]: 2024-04-27 02:39:09.343982743 +0000 UTC m=+0.018564650 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:39:09.738 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:09 smithi026 podman[130024]: 2024-04-27 02:39:09.471323048 +0000 UTC m=+0.145904941 container create 0eb9b20e747271d1e87b426cc20aa8ce4839a880ad6305635ef710e504aae51f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb) 2024-04-27T02:39:09.738 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:09 smithi026 podman[130024]: 2024-04-27 02:39:09.673874423 +0000 UTC m=+0.348456317 container init 0eb9b20e747271d1e87b426cc20aa8ce4839a880ad6305635ef710e504aae51f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-27T02:39:09.739 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:09 smithi026 podman[130024]: 2024-04-27 02:39:09.677133488 +0000 UTC m=+0.351715388 container start 0eb9b20e747271d1e87b426cc20aa8ce4839a880ad6305635ef710e504aae51f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-27T02:39:09.739 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:09 smithi026 podman[130024]: 2024-04-27 02:39:09.765478158 +0000 UTC m=+0.440060051 container attach 0eb9b20e747271d1e87b426cc20aa8ce4839a880ad6305635ef710e504aae51f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb) 2024-04-27T02:39:10.101 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:09 smithi026 podman[130024]: 2024-04-27 02:39:09.978075327 +0000 UTC m=+0.652657221 container died 0eb9b20e747271d1e87b426cc20aa8ce4839a880ad6305635ef710e504aae51f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:39:10.492 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.2.service' 2024-04-27T02:39:10.516 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:10 smithi026 podman[130050]: 2024-04-27 02:39:10.493678313 +0000 UTC m=+0.506506012 container remove 0eb9b20e747271d1e87b426cc20aa8ce4839a880ad6305635ef710e504aae51f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:39:10.516 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:10 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-27T02:39:10.516 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:10 smithi026 systemd[1]: Stopped Ceph osd.2 for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:39:10.516 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 27 02:39:10 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.2.service: Consumed 5.311s CPU time. 2024-04-27T02:39:10.531 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:10.531 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-27T02:39:10.532 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-27T02:39:10.532 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.3 2024-04-27T02:39:10.851 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:10 smithi026 systemd[1]: Stopping Ceph osd.3 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:11.351 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:10 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:39:10.907+0000 7fd23f8bf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:39:11.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:10 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:39:10.907+0000 7fd23f8bf700 -1 osd.3 158 *** Got signal Terminated *** 2024-04-27T02:39:11.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:10 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:39:10.907+0000 7fd23f8bf700 -1 osd.3 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:39:13.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:13 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:13.463+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:13.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:13.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:13.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:13.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:13.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:13 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:13.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:14.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:14 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:14.421+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:15.601 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:15 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3[112976]: 2024-04-27T02:39:15.202+0000 7fd23beb3700 -1 osd.3 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.466996+0000 front 2024-04-27T02:38:50.467017+0000 (oldest deadline 2024-04-27T02:39:14.467476+0000) 2024-04-27T02:39:15.842 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:15 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:15.374+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:15.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:15 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:15.457+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:16.319 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:16.058+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:16.546 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:16 smithi026 podman[130129]: 2024-04-27 02:39:16.261494626 +0000 UTC m=+5.473325389 container died a9b308864afa7d3b119146f3177c1053b6c031e0ae4a1735e664a6f7bea603f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5) 2024-04-27T02:39:16.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:16.455+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:16.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:16.504+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:16.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:16 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:16.345+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:16.837 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:16 smithi026 podman[130129]: 2024-04-27 02:39:16.646796608 +0000 UTC m=+5.858627379 container cleanup a9b308864afa7d3b119146f3177c1053b6c031e0ae4a1735e664a6f7bea603f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, ceph=True) 2024-04-27T02:39:16.837 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:16 smithi026 bash[130129]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3 2024-04-27T02:39:17.101 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:16 smithi026 podman[130147]: 2024-04-27 02:39:16.864458678 +0000 UTC m=+0.602488593 container remove a9b308864afa7d3b119146f3177c1053b6c031e0ae4a1735e664a6f7bea603f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:39:17.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:17.025+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:17.423 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:17 smithi026 podman[130241]: 2024-04-27 02:39:17.084575481 +0000 UTC m=+0.018874324 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:39:17.424 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:17 smithi026 podman[130241]: 2024-04-27 02:39:17.181971944 +0000 UTC m=+0.116270809 container create 73c745e0f42760ea3d6ef1914954dbbe3943747a53b55123c1eb8d31bc3721f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424) 2024-04-27T02:39:17.424 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:17 smithi026 podman[130241]: 2024-04-27 02:39:17.392469673 +0000 UTC m=+0.326768497 container init 73c745e0f42760ea3d6ef1914954dbbe3943747a53b55123c1eb8d31bc3721f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-04-27T02:39:17.424 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:17 smithi026 podman[130241]: 2024-04-27 02:39:17.395541905 +0000 UTC m=+0.329840738 container start 73c745e0f42760ea3d6ef1914954dbbe3943747a53b55123c1eb8d31bc3721f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-27T02:39:17.424 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:17 smithi026 podman[130241]: 2024-04-27 02:39:17.450707325 +0000 UTC m=+0.385006153 container attach 73c745e0f42760ea3d6ef1914954dbbe3943747a53b55123c1eb8d31bc3721f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:39:17.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:17.473+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:17.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:17.377+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:17.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:17.507+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:17.960 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 27 02:39:17 smithi026 podman[130241]: 2024-04-27 02:39:17.70519337 +0000 UTC m=+0.639492185 container died 73c745e0f42760ea3d6ef1914954dbbe3943747a53b55123c1eb8d31bc3721f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:39:18.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.3.service' 2024-04-27T02:39:18.277 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:18.277 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-27T02:39:18.277 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-27T02:39:18.277 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.4 2024-04-27T02:39:18.304 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:17 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:17.980+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:18.592 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:18 smithi151 systemd[1]: Stopping Ceph osd.4 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:18.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:18.480+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:18.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:18.400+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:18.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:18.535+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:18.940 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:18.622+0000 7f5e5a17c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:39:18.940 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:18.622+0000 7f5e5a17c700 -1 osd.4 158 *** Got signal Terminated *** 2024-04-27T02:39:18.940 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:18.622+0000 7f5e5a17c700 -1 osd.4 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:39:19.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:18 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:18.966+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:19.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:19.445+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:19.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:19.394+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:19.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:19 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:19.570+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:20.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:20.001+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:20.842 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:20.396+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:20.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:20.427+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:20.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:20 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:20.524+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:21.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:21.045+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:21.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:21.045+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:55.249593+0000 front 2024-04-27T02:38:55.249750+0000 (oldest deadline 2024-04-27T02:39:20.549482+0000) 2024-04-27T02:39:21.842 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:21.419+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:21.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:21.392+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:21.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:21.503+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:22.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:22.007+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:22.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:22.007+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:55.249593+0000 front 2024-04-27T02:38:55.249750+0000 (oldest deadline 2024-04-27T02:39:20.549482+0000) 2024-04-27T02:39:22.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:22.512+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:22.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:22.411+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:22.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:22.411+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:22.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:22.391+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:22.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:22 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:22.391+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:23.342 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:23.000+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:48.848925+0000 front 2024-04-27T02:38:48.849011+0000 (oldest deadline 2024-04-27T02:39:15.249149+0000) 2024-04-27T02:39:23.343 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4[75647]: 2024-04-27T02:39:23.000+0000 7f5e52768700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:55.249593+0000 front 2024-04-27T02:38:55.249750+0000 (oldest deadline 2024-04-27T02:39:20.549482+0000) 2024-04-27T02:39:23.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:23.511+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:23.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:23.456+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:23.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:23.456+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:23.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:23.426+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:23.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:23 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:23.426+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:23.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:23.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:23.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:23.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:23.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:23 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:23.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:24.304 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:23 smithi151 podman[99188]: 2024-04-27 02:39:23.943280145 +0000 UTC m=+5.438615588 container died 5f13d235f4f4d7f5ddd78cd4b89aadeacd88a0a3b0b35cdd4dcb018170efd03d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5) 2024-04-27T02:39:24.589 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:24.559+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:24.589 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:24 smithi151 podman[99188]: 2024-04-27 02:39:24.332024442 +0000 UTC m=+5.827359930 container cleanup 5f13d235f4f4d7f5ddd78cd4b89aadeacd88a0a3b0b35cdd4dcb018170efd03d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-27T02:39:24.590 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:24 smithi151 bash[99188]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4 2024-04-27T02:39:24.590 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:24.418+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:24.590 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:24.418+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:24.590 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:24.468+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:24.590 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:24 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:24.468+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:24.842 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:24 smithi151 podman[99207]: 2024-04-27 02:39:24.617145063 +0000 UTC m=+0.666481317 container remove 5f13d235f4f4d7f5ddd78cd4b89aadeacd88a0a3b0b35cdd4dcb018170efd03d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0) 2024-04-27T02:39:25.186 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:24 smithi151 podman[99299]: 2024-04-27 02:39:24.856256174 +0000 UTC m=+0.017968960 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:39:25.186 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:24 smithi151 podman[99299]: 2024-04-27 02:39:24.99366499 +0000 UTC m=+0.155377753 container create e89fc01fbff1f4ead637d5f95175514a30d03cf0f30d6c733e11a09bb9149aa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-27T02:39:25.186 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:25 smithi151 podman[99299]: 2024-04-27 02:39:25.213437304 +0000 UTC m=+0.375150067 container init e89fc01fbff1f4ead637d5f95175514a30d03cf0f30d6c733e11a09bb9149aa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-27T02:39:25.477 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:25.424+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:25.477 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:25.424+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:25.478 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:25 smithi151 podman[99299]: 2024-04-27 02:39:25.216979331 +0000 UTC m=+0.378692098 container start e89fc01fbff1f4ead637d5f95175514a30d03cf0f30d6c733e11a09bb9149aa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:39:25.478 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:25 smithi151 podman[99299]: 2024-04-27 02:39:25.305208106 +0000 UTC m=+0.466920872 container attach e89fc01fbff1f4ead637d5f95175514a30d03cf0f30d6c733e11a09bb9149aa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-27T02:39:25.478 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:25.427+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:25.478 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:25.427+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:25.773 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 27 02:39:25 smithi151 podman[99299]: 2024-04-27 02:39:25.505042105 +0000 UTC m=+0.666754888 container died e89fc01fbff1f4ead637d5f95175514a30d03cf0f30d6c733e11a09bb9149aa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:39:25.773 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:25.535+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:25.773 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:25 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:25.535+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:26.179 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.4.service' 2024-04-27T02:39:26.258 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:26.258 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-27T02:39:26.259 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-27T02:39:26.259 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.5 2024-04-27T02:39:26.415 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:26 smithi151 systemd[1]: Stopping Ceph osd.5 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:26.416 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:26.396+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:26.416 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:26.396+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:26.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:26.551+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:26.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:26.551+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:26.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:26.462+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:26.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:26.462+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:26.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:26.630+0000 7f749edb3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:39:26.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:26.630+0000 7f749edb3700 -1 osd.5 158 *** Got signal Terminated *** 2024-04-27T02:39:26.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:26 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:26.630+0000 7f749edb3700 -1 osd.5 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:39:27.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:27 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:27.413+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:27.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:27 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:27.413+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:27.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:27 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:27.509+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:27.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:27 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:27.509+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:27.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:27 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:27.448+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:27.593 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:27 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:27.448+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:28.842 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:28.458+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:28.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:28.458+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:28.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:28.395+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:28.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:28.395+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:28.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:28.503+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:28.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:28 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:28.503+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:29.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:29.507+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:29.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:29.507+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:29.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:29.499+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:29.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:29.499+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:29.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:29.422+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:29.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:29.422+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:29.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:29 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:29.422+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:30.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:30 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:30.515+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:30.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:30 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:30.515+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:30.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:30 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:30.519+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:30.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:30 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:30.519+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:30.844 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:30 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:30.466+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:30.844 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:30 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:30.466+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:30.844 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:30 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:30.466+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:31.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:31.554+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:31.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:31.554+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:31.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:31.531+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.951096+0000 front 2024-04-27T02:38:50.951183+0000 (oldest deadline 2024-04-27T02:39:14.451056+0000) 2024-04-27T02:39:31.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:31.531+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:59.152153+0000 front 2024-04-27T02:38:59.152136+0000 (oldest deadline 2024-04-27T02:39:22.052020+0000) 2024-04-27T02:39:31.843 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5[80667]: 2024-04-27T02:39:31.531+0000 7f749739f700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.352895+0000 front 2024-04-27T02:39:07.352841+0000 (oldest deadline 2024-04-27T02:39:31.452848+0000) 2024-04-27T02:39:31.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:31.509+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:31.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:31.509+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:31.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:31 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:31.509+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:32.247 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:31 smithi151 podman[99405]: 2024-04-27 02:39:31.953899285 +0000 UTC m=+5.439710939 container died 4d15baed129ee20422317e43200525054c008649d637420220bcdcc6c90aaf50 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=) 2024-04-27T02:39:32.517 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:32 smithi151 podman[99405]: 2024-04-27 02:39:32.456573112 +0000 UTC m=+5.942384808 container cleanup 4d15baed129ee20422317e43200525054c008649d637420220bcdcc6c90aaf50 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424) 2024-04-27T02:39:32.518 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:32 smithi151 bash[99405]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5 2024-04-27T02:39:32.518 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:32 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:32.505+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:32.518 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:32 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:32.505+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:32.518 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:32 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:32.505+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:32.842 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:32 smithi151 podman[99433]: 2024-04-27 02:39:32.700093413 +0000 UTC m=+0.737602303 container remove 4d15baed129ee20422317e43200525054c008649d637420220bcdcc6c90aaf50 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-27T02:39:32.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:32 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:32.544+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:32.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:32 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:32.544+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:32.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:32 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:32.544+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:33.301 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:33 smithi151 podman[99526]: 2024-04-27 02:39:32.935640551 +0000 UTC m=+0.017833372 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:39:33.302 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:33 smithi151 podman[99526]: 2024-04-27 02:39:33.034510603 +0000 UTC m=+0.116703468 container create 7431b26c6f91140d81267b75c36e1316dde73f3b466136affba9c2575ce9fcca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-27T02:39:33.302 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:33 smithi151 podman[99526]: 2024-04-27 02:39:33.245697014 +0000 UTC m=+0.327889837 container init 7431b26c6f91140d81267b75c36e1316dde73f3b466136affba9c2575ce9fcca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-27T02:39:33.302 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:33 smithi151 podman[99526]: 2024-04-27 02:39:33.248782033 +0000 UTC m=+0.330974864 container start 7431b26c6f91140d81267b75c36e1316dde73f3b466136affba9c2575ce9fcca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb) 2024-04-27T02:39:33.302 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:33 smithi151 podman[99526]: 2024-04-27 02:39:33.329135966 +0000 UTC m=+0.411328795 container attach 7431b26c6f91140d81267b75c36e1316dde73f3b466136affba9c2575ce9fcca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424) 2024-04-27T02:39:33.592 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 27 02:39:33 smithi151 podman[99526]: 2024-04-27 02:39:33.542834883 +0000 UTC m=+0.625027710 container died 7431b26c6f91140d81267b75c36e1316dde73f3b466136affba9c2575ce9fcca (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:39:33.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:33.568+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:33.593 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:33.568+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:33.593 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:33.568+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:33.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:33.514+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:33.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:33.514+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:33.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:33 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:33.514+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:33.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:33.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:33.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:33.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:33.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:33 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:33.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:34.227 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.5.service' 2024-04-27T02:39:34.270 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:34.271 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-27T02:39:34.271 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-27T02:39:34.271 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.6 2024-04-27T02:39:34.483 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:34 smithi151 systemd[1]: Stopping Ceph osd.6 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:34.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:34.509+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:34.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:34.509+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:34.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:34.509+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:34.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:34.592+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:34.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:34.592+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:34.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:34.592+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:34.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:34.628+0000 7f6d3624e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:39:34.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:34.628+0000 7f6d3624e700 -1 osd.6 158 *** Got signal Terminated *** 2024-04-27T02:39:34.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:34 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:34.628+0000 7f6d3624e700 -1 osd.6 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:39:35.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:35 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:35.571+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:35.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:35 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:35.571+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:35.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:35 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:35.571+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:35.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:35 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:35.521+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:35.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:35 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:35.521+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:35.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:35 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:35.521+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:36.825 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:36.531+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:36.825 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:36.531+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:36.825 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:36.531+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:36.825 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:36.489+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:36.825 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:36.489+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:36.825 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:36 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:36.489+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:37.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:37.510+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:37.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:37.510+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:37.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:37.510+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:37.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:37.469+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:37.844 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:37.469+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:37.844 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:37 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:37.469+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:38.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:38.488+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:38.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:38.488+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:38.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:38.488+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:38.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:38.488+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:15.601432+0000 front 2024-04-27T02:39:15.601495+0000 (oldest deadline 2024-04-27T02:39:37.801879+0000) 2024-04-27T02:39:38.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:38.427+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:38.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:38.427+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:38.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:38.427+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:38.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:38 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:38.427+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:39.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:39.462+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:50.998630+0000 front 2024-04-27T02:38:50.998663+0000 (oldest deadline 2024-04-27T02:39:16.298621+0000) 2024-04-27T02:39:39.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:39.462+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.299309+0000 front 2024-04-27T02:38:56.299229+0000 (oldest deadline 2024-04-27T02:39:22.199076+0000) 2024-04-27T02:39:39.842 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:39.462+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:05.700190+0000 front 2024-04-27T02:39:05.700205+0000 (oldest deadline 2024-04-27T02:39:29.200040+0000) 2024-04-27T02:39:39.843 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6[85505]: 2024-04-27T02:39:39.462+0000 7f6d2e83a700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:15.601432+0000 front 2024-04-27T02:39:15.601495+0000 (oldest deadline 2024-04-27T02:39:37.801879+0000) 2024-04-27T02:39:39.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:39.422+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:39.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:39.422+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:39.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:39.422+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:39.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:39 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:39.422+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:40.236 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:40 smithi151 podman[99631]: 2024-04-27 02:39:40.00146185 +0000 UTC m=+5.472646832 container died 88ba76966dafe2f721e99abc4dbb5fe237b30e0ccd87b42519b412180f688704 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-27T02:39:40.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:40 smithi151 podman[99631]: 2024-04-27 02:39:40.421341107 +0000 UTC m=+5.892526078 container cleanup 88ba76966dafe2f721e99abc4dbb5fe237b30e0ccd87b42519b412180f688704 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-27T02:39:40.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:40 smithi151 bash[99631]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6 2024-04-27T02:39:40.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:40.431+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:40.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:40.431+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:40.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:40.431+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:40.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:40 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:40.431+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:40.932 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:40 smithi151 podman[99650]: 2024-04-27 02:39:40.6361053 +0000 UTC m=+0.627084150 container remove 88ba76966dafe2f721e99abc4dbb5fe237b30e0ccd87b42519b412180f688704 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:39:41.232 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:40 smithi151 podman[99744]: 2024-04-27 02:39:40.862879819 +0000 UTC m=+0.018164591 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:39:41.233 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:40 smithi151 podman[99744]: 2024-04-27 02:39:40.965805916 +0000 UTC m=+0.121090727 container create 6727b522148742ba8d75ca1e4415c09570a2e050cd9c1d25550ee8439b39e32f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-27T02:39:41.233 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:41 smithi151 podman[99744]: 2024-04-27 02:39:41.176936687 +0000 UTC m=+0.332221455 container init 6727b522148742ba8d75ca1e4415c09570a2e050cd9c1d25550ee8439b39e32f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-27T02:39:41.233 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:41 smithi151 podman[99744]: 2024-04-27 02:39:41.18000855 +0000 UTC m=+0.335293319 container start 6727b522148742ba8d75ca1e4415c09570a2e050cd9c1d25550ee8439b39e32f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:39:41.233 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:41 smithi151 podman[99744]: 2024-04-27 02:39:41.260484331 +0000 UTC m=+0.415769113 container attach 6727b522148742ba8d75ca1e4415c09570a2e050cd9c1d25550ee8439b39e32f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-27T02:39:41.592 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:41 smithi151 podman[99744]: 2024-04-27 02:39:41.46535107 +0000 UTC m=+0.620635831 container died 6727b522148742ba8d75ca1e4415c09570a2e050cd9c1d25550ee8439b39e32f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True) 2024-04-27T02:39:41.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:41.386+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:41.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:41.386+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:41.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:41.386+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:41.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:41 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:41.386+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:41.983 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 27 02:39:41 smithi151 podman[99770]: 2024-04-27 02:39:41.989443695 +0000 UTC m=+0.518896935 container remove 6727b522148742ba8d75ca1e4415c09570a2e050cd9c1d25550ee8439b39e32f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-27T02:39:41.988 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.6.service' 2024-04-27T02:39:42.025 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:42.025 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-27T02:39:42.025 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-27T02:39:42.026 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.7 2024-04-27T02:39:42.342 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 systemd[1]: Stopping Ceph osd.7 for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:42.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:42.375+0000 7f4f8da69700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-27T02:39:42.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:42.375+0000 7f4f8da69700 -1 osd.7 158 *** Got signal Terminated *** 2024-04-27T02:39:42.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:42.375+0000 7f4f8da69700 -1 osd.7 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-27T02:39:42.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:42.418+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:42.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:42.418+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:42.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:42.418+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:42.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:42 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:42.418+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:43.744 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:43.435+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:43.744 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:43.435+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:43.744 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:43.435+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:43.744 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:43 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:43.435+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:43.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:43.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:43.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:43.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:43.851 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:43 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:43.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:44.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:44.451+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:44.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:44.451+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:44.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:44.451+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:44.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:44 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:44.451+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:45.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:45 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:45.432+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:45.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:45 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:45.432+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:45.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:45 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:45.432+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:45.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:45 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:45.432+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:46.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:46.415+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-27T02:38:49.085815+0000 front 2024-04-27T02:38:49.085772+0000 (oldest deadline 2024-04-27T02:39:13.185575+0000) 2024-04-27T02:39:46.842 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:46.415+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-27T02:38:56.086727+0000 front 2024-04-27T02:38:56.086610+0000 (oldest deadline 2024-04-27T02:39:25.487300+0000) 2024-04-27T02:39:46.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:46.415+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-27T02:39:07.188447+0000 front 2024-04-27T02:39:07.188339+0000 (oldest deadline 2024-04-27T02:39:32.488262+0000) 2024-04-27T02:39:46.843 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:46 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7[90189]: 2024-04-27T02:39:46.415+0000 7f4f86055700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-27T02:39:12.488789+0000 front 2024-04-27T02:39:12.488843+0000 (oldest deadline 2024-04-27T02:39:37.689177+0000) 2024-04-27T02:39:48.257 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:47 smithi151 podman[99850]: 2024-04-27 02:39:47.880619923 +0000 UTC m=+5.606643044 container died bec04d87416c4d23463803ae25bceb5650b301ffa2656d802f92371b8e1e48ef (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-27T02:39:48.257 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:48 smithi151 podman[99850]: 2024-04-27 02:39:48.285003899 +0000 UTC m=+6.011027022 container cleanup bec04d87416c4d23463803ae25bceb5650b301ffa2656d802f92371b8e1e48ef (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-27T02:39:48.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:48 smithi151 bash[99850]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7 2024-04-27T02:39:48.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:48 smithi151 podman[99879]: 2024-04-27 02:39:48.495107712 +0000 UTC m=+0.607760702 container remove bec04d87416c4d23463803ae25bceb5650b301ffa2656d802f92371b8e1e48ef (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-27T02:39:49.092 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:48 smithi151 podman[99977]: 2024-04-27 02:39:48.82111905 +0000 UTC m=+0.106556077 container create 3fd2581c111fdbc2a2147cc69f699c6ca13c32bdcbfaad1caede411f2c74d3f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-27T02:39:49.092 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:48 smithi151 podman[99977]: 2024-04-27 02:39:48.732582059 +0000 UTC m=+0.018019061 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-27T02:39:49.092 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:49 smithi151 podman[99977]: 2024-04-27 02:39:49.041059917 +0000 UTC m=+0.326496919 container init 3fd2581c111fdbc2a2147cc69f699c6ca13c32bdcbfaad1caede411f2c74d3f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-27T02:39:49.093 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:49 smithi151 podman[99977]: 2024-04-27 02:39:49.044496779 +0000 UTC m=+0.329933765 container start 3fd2581c111fdbc2a2147cc69f699c6ca13c32bdcbfaad1caede411f2c74d3f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-27T02:39:49.592 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:49 smithi151 podman[99977]: 2024-04-27 02:39:49.123806547 +0000 UTC m=+0.409243548 container attach 3fd2581c111fdbc2a2147cc69f699c6ca13c32bdcbfaad1caede411f2c74d3f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-27T02:39:49.593 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 27 02:39:49 smithi151 podman[99977]: 2024-04-27 02:39:49.337451866 +0000 UTC m=+0.622888845 container died 3fd2581c111fdbc2a2147cc69f699c6ca13c32bdcbfaad1caede411f2c74d3f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS) 2024-04-27T02:39:49.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@osd.7.service' 2024-04-27T02:39:49.907 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:49.910 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-27T02:39:49.910 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-27T02:39:49.910 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a 2024-04-27T02:39:50.204 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 systemd[1]: Stopping Ceph prometheus.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:50.522 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.231Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:39:50.522 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.231Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:39:50.522 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.231Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:39:50.522 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.232Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:39:50.522 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.232Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-27T02:39:50.522 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.272Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.272Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.272Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.272Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.272Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.272Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.273Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.273Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.274Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.275Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.275Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a[61307]: ts=2024-04-27T02:39:50.275Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-27T02:39:50.523 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 podman[100081]: 2024-04-27 02:39:50.283251882 +0000 UTC m=+0.119438788 container died 53ac67210d3bfe24663fcd845ed2eec096eb0c61b989e1e0a839a0e16ce930f3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:50.842 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 podman[100081]: 2024-04-27 02:39:50.639926952 +0000 UTC m=+0.476113860 container cleanup 53ac67210d3bfe24663fcd845ed2eec096eb0c61b989e1e0a839a0e16ce930f3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:50.842 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 27 02:39:50 smithi151 bash[100081]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-prometheus-a 2024-04-27T02:39:51.041 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7a5462be-0439-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-27T02:39:51.074 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-27T02:39:51.074 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-27T02:39:51.075 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7a5462be-0439-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-27T02:39:52.879 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:52 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:53.201 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:52 smithi026 podman[130633]: 2024-04-27 02:39:52.90613962 +0000 UTC m=+0.115470744 container died a01cf397c18aebf5e6684228738549590ef6a7bbde3a9dcacfaa8f7615e26ce8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:53.201 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:53 smithi026 podman[130633]: 2024-04-27 02:39:53.229298146 +0000 UTC m=+0.438629281 container cleanup a01cf397c18aebf5e6684228738549590ef6a7bbde3a9dcacfaa8f7615e26ce8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:53.534 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:53 smithi026 bash[130633]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a 2024-04-27T02:39:53.534 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:53 smithi026 podman[130647]: 2024-04-27 02:39:53.380336328 +0000 UTC m=+0.478334635 container remove a01cf397c18aebf5e6684228738549590ef6a7bbde3a9dcacfaa8f7615e26ce8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:53.534 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:53 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:39:53.851 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:53 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-27T02:39:53.851 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:53 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:39:53.851 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 27 02:39:53 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.245s CPU time. 2024-04-27T02:39:53.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:53.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=5 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:53.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:53.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-27T02:39:53.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:53 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:53.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-27T02:39:53.852 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:53 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:39:54.349 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a[77436]: ts=2024-04-27T02:39:54.050Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-27T02:39:54.349 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 podman[130752]: 2024-04-27 02:39:54.091030535 +0000 UTC m=+0.147217516 container died 8b1e48c88073747df63e3d5b8490a6f4bc9263066cc5dd83a9f631d3f7214107 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:54.601 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 podman[130752]: 2024-04-27 02:39:54.47549751 +0000 UTC m=+0.531684479 container cleanup 8b1e48c88073747df63e3d5b8490a6f4bc9263066cc5dd83a9f631d3f7214107 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:54.601 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 bash[130752]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a 2024-04-27T02:39:54.950 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 podman[130770]: 2024-04-27 02:39:54.726513261 +0000 UTC m=+0.628273107 container remove 8b1e48c88073747df63e3d5b8490a6f4bc9263066cc5dd83a9f631d3f7214107 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-27T02:39:54.950 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 podman[130770]: 2024-04-27 02:39:54.802182968 +0000 UTC m=+0.703942817 volume remove f0ed0d89250ec2855f85c96bfa8c7afc31fd8a377fafc25aa37b58d732b21f98 2024-04-27T02:39:54.950 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-27T02:39:54.950 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:39:54.950 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 27 02:39:54 smithi026 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.670s CPU time. 2024-04-27T02:40:18.048 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7a5462be-0439-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-27T02:40:19.784 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:19 smithi151 systemd[1]: Stopping Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:40:20.058 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:19 smithi151 podman[100495]: 2024-04-27 02:40:19.811828172 +0000 UTC m=+0.099902345 container died 881b4b216b6ba53a2fcf123061cdd3d4ef5db9a1d05f0be7079fbaa2edeb78de (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:40:20.342 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:20 smithi151 podman[100495]: 2024-04-27 02:40:20.233936 +0000 UTC m=+0.522010169 container cleanup 881b4b216b6ba53a2fcf123061cdd3d4ef5db9a1d05f0be7079fbaa2edeb78de (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:40:20.342 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:20 smithi151 bash[100495]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b 2024-04-27T02:40:20.699 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:20 smithi151 podman[100509]: 2024-04-27 02:40:20.461770751 +0000 UTC m=+0.650873843 container remove 881b4b216b6ba53a2fcf123061cdd3d4ef5db9a1d05f0be7079fbaa2edeb78de (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-27T02:40:20.699 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:20 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-27T02:40:20.699 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:20 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-27T02:40:20.699 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:20 smithi151 systemd[1]: Stopped Ceph node-exporter.b for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:40:20.699 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 27 02:40:20 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.094s CPU time. 2024-04-27T02:40:21.092 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:20 smithi151 systemd[1]: Stopping Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968... 2024-04-27T02:40:21.388 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=server t=2024-04-27T02:40:21.140191795Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-04-27T02:40:21.389 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=tracing t=2024-04-27T02:40:21.140584368Z level=info msg="Closing tracing" 2024-04-27T02:40:21.389 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a[63281]: logger=ticker t=2024-04-27T02:40:21.140585422Z level=info msg=stopped last_tick=2024-04-27T02:40:20Z 2024-04-27T02:40:21.389 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 podman[100619]: 2024-04-27 02:40:21.150436206 +0000 UTC m=+0.126530850 container died 89665f31c1488acfce7a5dd13a3eb9faba4ed0555df3ec0e01811c122b800e86 (image=quay.io/ceph/grafana:9.4.12, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-27T02:40:21.830 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 podman[100619]: 2024-04-27 02:40:21.481951746 +0000 UTC m=+0.458046401 container cleanup 89665f31c1488acfce7a5dd13a3eb9faba4ed0555df3ec0e01811c122b800e86 (image=quay.io/ceph/grafana:9.4.12, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-27T02:40:21.830 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 bash[100619]: ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a 2024-04-27T02:40:21.830 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 podman[100634]: 2024-04-27 02:40:21.658233454 +0000 UTC m=+0.509476701 container remove 89665f31c1488acfce7a5dd13a3eb9faba4ed0555df3ec0e01811c122b800e86 (image=quay.io/ceph/grafana:9.4.12, name=ceph-7a5462be-0439-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-27T02:40:22.259 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-27T02:40:22.259 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 systemd[1]: Stopped Ceph grafana.a for 7a5462be-0439-11ef-bc93-c7b262605968. 2024-04-27T02:40:22.259 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 27 02:40:21 smithi151 systemd[1]: ceph-7a5462be-0439-11ef-bc93-c7b262605968@grafana.a.service: Consumed 11.089s CPU time. 2024-04-27T02:40:33.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-27T02:40:33.529 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-27T02:40:33.555 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-27T02:40:33.557 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530/remote/smithi026/crash 2024-04-27T02:40:33.558 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/crash -- . 2024-04-27T02:40:33.612 INFO:teuthology.orchestra.run.smithi026.stderr:tar: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-27T02:40:33.612 INFO:teuthology.orchestra.run.smithi026.stderr:tar: Error is not recoverable: exiting now 2024-04-27T02:40:33.614 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530/remote/smithi151/crash 2024-04-27T02:40:33.615 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/crash -- . 2024-04-27T02:40:33.653 INFO:teuthology.orchestra.run.smithi151.stderr:tar: /var/lib/ceph/7a5462be-0439-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-27T02:40:33.653 INFO:teuthology.orchestra.run.smithi151.stderr:tar: Error is not recoverable: exiting now 2024-04-27T02:40:33.654 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-27T02:40:33.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-27T02:40:33.684 INFO:teuthology.orchestra.run.smithi026.stdout:2024-04-27T02:12:38.282810+0000 mon.a (mon.0) 967 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:40:33.684 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-27T02:40:33.684 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[SEC\]' /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-27T02:40:33.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[ERR\]' /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-27T02:40:33.822 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[WRN\]' /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-27T02:40:33.888 INFO:teuthology.orchestra.run.smithi026.stdout:2024-04-27T02:12:38.282810+0000 mon.a (mon.0) 967 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-27T02:40:33.889 INFO:tasks.cephadm:Compressing logs... 2024-04-27T02:40:33.889 DEBUG:teuthology.orchestra.run.smithi026:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-27T02:40:33.931 DEBUG:teuthology.orchestra.run.smithi151:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-27T02:40:33.954 INFO:teuthology.orchestra.run.smithi026.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-27T02:40:33.954 INFO:teuthology.orchestra.run.smithi026.stderr:: No such file or directory 2024-04-27T02:40:33.954 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log 2024-04-27T02:40:33.954 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-27T02:40:33.954 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-27T02:40:33.954 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-27T02:40:33.955 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-27T02:40:33.955 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-27T02:40:33.955 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-27T02:40:33.958 INFO:teuthology.orchestra.run.smithi151.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log: No such file or directory 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr: 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi151.blhads.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi151.blhads.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log: 75.7% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi151.blhads.log.gz 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-27T02:40:33.959 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-27T02:40:33.960 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.1.log: /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mgr.y.log: 93.8% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-27T02:40:33.961 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi026.oulbnt.log 2024-04-27T02:40:33.962 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.6.log: 88.9% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-27T02:40:33.962 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-27T02:40:33.963 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.cephadm.log: 90.4% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-27T02:40:33.964 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-27T02:40:33.964 INFO:teuthology.orchestra.run.smithi151.stderr: 91.1% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.audit.log.gz/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-volume.log: 2024-04-27T02:40:33.966 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.0.log: 94.4% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-27T02:40:33.966 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-27T02:40:33.967 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi026.oulbnt.log: 75.7% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi026.oulbnt.log.gz 2024-04-27T02:40:33.967 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-27T02:40:33.970 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.c.log: 92.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-27T02:40:33.970 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-27T02:40:33.979 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.b.log: 91.0% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-27T02:40:33.988 INFO:teuthology.orchestra.run.smithi151.stderr: 90.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-27T02:40:33.988 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/tcmu-runner.log 2024-04-27T02:40:33.991 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.cephadm.log: 94.2% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-27T02:40:34.004 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-27T02:40:34.004 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/tcmu-runner.log: 89.1% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/tcmu-runner.log.gz 2024-04-27T02:40:34.051 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-volume.log: 91.0% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-27T02:40:34.076 INFO:teuthology.orchestra.run.smithi151.stderr: 89.7% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-27T02:40:34.480 INFO:teuthology.orchestra.run.smithi026.stderr: 89.1% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-27T02:40:34.750 INFO:teuthology.orchestra.run.smithi026.stderr: 92.4% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-27T02:40:34.822 INFO:teuthology.orchestra.run.smithi151.stderr: 92.5% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-27T02:40:35.459 INFO:teuthology.orchestra.run.smithi151.stderr: 93.2% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-27T02:40:35.739 INFO:teuthology.orchestra.run.smithi026.stderr: 91.0% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-27T02:40:35.788 INFO:teuthology.orchestra.run.smithi026.stderr: 93.3% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-27T02:40:35.837 INFO:teuthology.orchestra.run.smithi151.stderr: 93.4% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-27T02:40:35.849 INFO:teuthology.orchestra.run.smithi151.stderr: 93.8% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-27T02:40:36.222 INFO:teuthology.orchestra.run.smithi026.stderr: 93.5% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-27T02:40:36.325 INFO:teuthology.orchestra.run.smithi151.stderr: 93.6% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-27T02:40:36.328 INFO:teuthology.orchestra.run.smithi151.stderr: 2024-04-27T02:40:36.328 INFO:teuthology.orchestra.run.smithi151.stderr:real 0m2.381s 2024-04-27T02:40:36.328 INFO:teuthology.orchestra.run.smithi151.stderr:user 0m8.335s 2024-04-27T02:40:36.328 INFO:teuthology.orchestra.run.smithi151.stderr:sys 0m0.353s 2024-04-27T02:40:36.395 INFO:teuthology.orchestra.run.smithi026.stderr: 93.4% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-27T02:40:36.784 INFO:teuthology.orchestra.run.smithi026.stderr: 93.4% -- replaced with /var/log/ceph/7a5462be-0439-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-27T02:40:36.786 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-27T02:40:36.787 INFO:teuthology.orchestra.run.smithi026.stderr:real 0m2.843s 2024-04-27T02:40:36.787 INFO:teuthology.orchestra.run.smithi026.stderr:user 0m12.028s 2024-04-27T02:40:36.787 INFO:teuthology.orchestra.run.smithi026.stderr:sys 0m0.430s 2024-04-27T02:40:36.787 INFO:tasks.cephadm:Archiving logs... 2024-04-27T02:40:36.787 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530/remote/smithi026/log 2024-04-27T02:40:36.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-27T02:40:38.320 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530/remote/smithi151/log 2024-04-27T02:40:38.321 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-27T02:40:39.510 INFO:tasks.cephadm:Removing cluster... 2024-04-27T02:40:39.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7a5462be-0439-11ef-bc93-c7b262605968 --force 2024-04-27T02:40:39.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7a5462be-0439-11ef-bc93-c7b262605968 --force 2024-04-27T02:40:40.106 INFO:tasks.cephadm:Removing cephadm ... 2024-04-27T02:40:40.107 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-27T02:40:40.123 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-27T02:40:40.138 INFO:tasks.cephadm:Teardown complete 2024-04-27T02:40:40.138 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-27T02:40:40.149 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-27T02:40:40.149 DEBUG:teuthology.orchestra.run.smithi026:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-27T02:40:40.165 DEBUG:teuthology.orchestra.run.smithi151:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-27T02:40:40.177 INFO:teuthology.orchestra.run.smithi026.stderr:bash: line 1: ntpq: command not found 2024-04-27T02:40:40.183 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-27T02:40:40.183 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2024-04-27T02:40:40.183 INFO:teuthology.orchestra.run.smithi026.stdout:^* hv01.front.sepia.ceph.com 3 6 377 9 +2193us[+2183us] +/- 44ms 2024-04-27T02:40:40.183 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 6 -2924us[-2924us] +/- 60ms 2024-04-27T02:40:40.183 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 8 +454us[ +454us] +/- 85ms 2024-04-27T02:40:40.183 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T02:40:40.191 INFO:teuthology.orchestra.run.smithi151.stderr:bash: line 1: ntpq: command not found 2024-04-27T02:40:40.195 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-27T02:40:40.195 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2024-04-27T02:40:40.195 INFO:teuthology.orchestra.run.smithi151.stdout:^* hv01.front.sepia.ceph.com 3 6 377 9 +2149us[+2144us] +/- 44ms 2024-04-27T02:40:40.195 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv02.front.sepia.ceph.com 3 8 377 75 -2918us[-2931us] +/- 59ms 2024-04-27T02:40:40.195 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 7 +398us[ +398us] +/- 85ms 2024-04-27T02:40:40.195 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-27T02:40:40.197 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-27T02:40:40.205 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-27T02:40:40.206 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-27T02:40:40.219 DEBUG:teuthology.orchestra.run.smithi026:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-27T02:40:40.312 DEBUG:teuthology.orchestra.run.smithi151:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-27T02:40:40.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-27T02:40:40.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T02:40:40.413 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-27T02:40:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T02:40:40.440 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-27T02:40:40.450 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-27T02:40:40.461 INFO:teuthology.task.internal:Duration was 2815.056126 seconds 2024-04-27T02:40:40.462 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-27T02:40:40.472 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-27T02:40:40.472 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-27T02:40:40.475 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-27T02:40:40.510 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-27T02:40:40.516 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-27T02:40:40.896 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-27T02:40:40.896 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi026.front.sepia.ceph.com 2024-04-27T02:40:40.896 DEBUG:teuthology.orchestra.run.smithi026:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-27T02:40:40.959 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi151.front.sepia.ceph.com 2024-04-27T02:40:40.959 DEBUG:teuthology.orchestra.run.smithi151:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-27T02:40:40.991 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-27T02:40:40.991 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-27T02:40:41.001 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-27T02:40:41.060 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-27T02:40:41.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-27T02:40:41.069 DEBUG:teuthology.orchestra.run.smithi151:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-27T02:40:41.661 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-27T02:40:41.673 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-27T02:40:41.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-27T02:40:41.698 DEBUG:teuthology.orchestra.run.smithi151:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-27T02:40:41.723 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-27T02:40:41.736 DEBUG:teuthology.orchestra.run.smithi026:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-27T02:40:41.741 DEBUG:teuthology.orchestra.run.smithi151:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-27T02:40:41.761 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = core 2024-04-27T02:40:41.785 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = core 2024-04-27T02:40:41.805 DEBUG:teuthology.orchestra.run.smithi026:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-27T02:40:41.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T02:40:41.834 DEBUG:teuthology.orchestra.run.smithi151:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-27T02:40:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-27T02:40:41.859 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-27T02:40:41.872 INFO:teuthology.task.internal:Transferring archived files... 2024-04-27T02:40:41.873 DEBUG:teuthology.misc:Transferring archived files from smithi026:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530/remote/smithi026 2024-04-27T02:40:41.873 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-27T02:40:41.925 DEBUG:teuthology.misc:Transferring archived files from smithi151:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530/remote/smithi151 2024-04-27T02:40:41.926 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-27T02:40:41.965 INFO:teuthology.task.internal:Removing archive directory... 2024-04-27T02:40:41.965 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-27T02:40:41.967 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-27T02:40:42.014 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-27T02:40:42.027 INFO:teuthology.task.internal:Not uploading archives. 2024-04-27T02:40:42.027 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-27T02:40:42.043 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-27T02:40:42.043 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-27T02:40:42.045 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-27T02:40:42.058 INFO:teuthology.orchestra.run.smithi026.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 27 02:40 /home/ubuntu/cephtest 2024-04-27T02:40:42.068 INFO:teuthology.orchestra.run.smithi151.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 27 02:40 /home/ubuntu/cephtest 2024-04-27T02:40:42.069 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-27T02:40:42.082 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-27T02:40:42.114 INFO:teuthology.nuke:Checking targets against current locks 2024-04-27T02:40:42.141 DEBUG:teuthology.nuke:shortname: smithi026 2024-04-27T02:40:42.142 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-27T02:40:42.165 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-27 01:48:00.132522', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEOKYw2n/szPDKE7tDEvlUO83CM+TUCJziUZclbG41ILu4TsHqqGHvRjg81llNjlYlm2nBJp5cE2ExDlo3hmBo8='} 2024-04-27T02:40:42.169 DEBUG:teuthology.nuke:shortname: smithi151 2024-04-27T02:40:42.169 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-27T02:40:42.194 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_21:08:03-orch-squid-distro-default-smithi/7675530', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-27 01:48:00.133866', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL49wT1Z7fqXO+J0pLl+liUsB0jvEFn0w9ec2PEMaNsTR2YnYH8Qo3tU94ihcS0t5TnvIA7y2uIT7595p8dLfJo='} 2024-04-27T02:40:42.222 INFO:teuthology.orchestra.console.smithi026:Power off 2024-04-27T02:40:42.222 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-27T02:40:42.248 DEBUG:teuthology.orchestra.console.smithi026:power off output: Chassis Power Control: Down/Off 2024-04-27T02:40:42.249 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-27T02:40:42.264 INFO:teuthology.orchestra.console.smithi151:Power off 2024-04-27T02:40:42.264 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-27T02:40:42.274 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2024-04-27T02:40:42.288 DEBUG:teuthology.orchestra.console.smithi151:power off output: Chassis Power Control: Down/Off 2024-04-27T02:40:42.288 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-27T02:40:42.314 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2024-04-27T02:40:46.274 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-27T02:40:46.315 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-27T02:40:46.384 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2024-04-27T02:40:46.426 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2024-04-27T02:40:50.389 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-27T02:40:50.427 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-27T02:40:50.500 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is off 2024-04-27T02:40:50.536 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2024-04-27T02:40:50.602 INFO:teuthology.orchestra.console.smithi026:Power off completed 2024-04-27T02:40:54.541 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-27T02:40:54.651 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is off 2024-04-27T02:40:54.752 INFO:teuthology.orchestra.console.smithi151:Power off completed 2024-04-27T02:40:54.856 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} duration: 2815.056125640869 failure_reason: '"2024-04-27T02:12:38.282810+0000 mon.a (mon.0) 967 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-04-27T02:40:54.856 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-27T02:40:54.941 INFO:teuthology.run:FAIL